
<!doctype html>
<html lang="en" class="no-js">
  <head>
    
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">
      
      
      
        <link rel="canonical" href="https://pytorch-widedeep.readthedocs.io/pytorch-widedeep/preprocessing.html">
      
      
        <link rel="prev" href="utils/text_utils.html">
      
      
        <link rel="next" href="load_from_folder.html">
      
      
      <link rel="icon" href="../assets/images/favicon.ico">
      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.5.43">
    
    
      
        <title>Preprocessing - pytorch_widedeep</title>
      
    
    
      <link rel="stylesheet" href="../assets/stylesheets/main.0253249f.min.css">
      
        
        <link rel="stylesheet" href="../assets/stylesheets/palette.06af60db.min.css">
      
      


    
    
      
    
    
      
        
        
        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
      
    
    
      <link rel="stylesheet" href="../assets/_mkdocstrings.css">
    
      <link rel="stylesheet" href="../stylesheets/extra.css">
    
    <script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
    
      

    
    
    
  </head>
  
  
    
    
      
    
    
    
    
    <body dir="ltr" data-md-color-scheme="default" data-md-color-primary="red" data-md-color-accent="deep-orange">
  
    
    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">
      
        
        <a href="#the-preprocessing-module" class="md-skip">
          Skip to content
        </a>
      
    </div>
    <div data-md-component="announce">
      
    </div>
    
    
      

  

<header class="md-header md-header--shadow md-header--lifted" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="../index.html" title="pytorch_widedeep" class="md-header__button md-logo" aria-label="pytorch_widedeep" data-md-component="logo">
      
  <img src="../assets/images/widedeep_logo.png" alt="logo">

    </a>
    <label class="md-header__button md-icon" for="__drawer">
      
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            pytorch_widedeep
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">
            
              Preprocessing
            
          </span>
        </div>
      </div>
    </div>
    
      
        <form class="md-header__option" data-md-component="palette">
  
    
    
    
    <input class="md-option" data-md-color-media="" data-md-color-scheme="default" data-md-color-primary="red" data-md-color-accent="deep-orange"  aria-label="Switch to dark mode"  type="radio" name="__palette" id="__palette_0">
    
      <label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_1" hidden>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a4 4 0 0 0-4 4 4 4 0 0 0 4 4 4 4 0 0 0 4-4 4 4 0 0 0-4-4m0 10a6 6 0 0 1-6-6 6 6 0 0 1 6-6 6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
      </label>
    
  
    
    
    
    <input class="md-option" data-md-color-media="" data-md-color-scheme="slate" data-md-color-primary="red" data-md-color-accent="deep-orange"  aria-label="Switch to light mode"  type="radio" name="__palette" id="__palette_1">
    
      <label class="md-header__button md-icon" title="Switch to light mode" for="__palette_0" hidden>
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 18c-.89 0-1.74-.2-2.5-.55C11.56 16.5 13 14.42 13 12s-1.44-4.5-3.5-5.45C10.26 6.2 11.11 6 12 6a6 6 0 0 1 6 6 6 6 0 0 1-6 6m8-9.31V4h-4.69L12 .69 8.69 4H4v4.69L.69 12 4 15.31V20h4.69L12 23.31 15.31 20H20v-4.69L23.31 12z"/></svg>
      </label>
    
  
</form>
      
    
    
      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
    
    
    
      <label class="md-header__button md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
        
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">
        
        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
          
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
        </button>
      </nav>
      
    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list" role="presentation"></ol>
        </div>
      </div>
    </div>
  </div>
</div>
    
    
      <div class="md-header__source">
        <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
  </div>
  <div class="md-source__repository">
    pytorch_widedeep
  </div>
</a>
      </div>
    
  </nav>
  
    
      
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
  <div class="md-grid">
    <ul class="md-tabs__list">
      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../index.html" class="md-tabs__link">
        
  
    
  
  Home

      </a>
    </li>
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../installation.html" class="md-tabs__link">
        
  
    
  
  Installation

      </a>
    </li>
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../quick_start.html" class="md-tabs__link">
        
  
    
  
  Quick Start

      </a>
    </li>
  

      
        
  
  
    
  
  
    
    
      
  
  
    
  
  
    
    
      <li class="md-tabs__item md-tabs__item--active">
        <a href="utils/index.html" class="md-tabs__link">
          
  
    
  
  Pytorch-widedeep

        </a>
      </li>
    
  

    
  

      
        
  
  
  
    
    
      <li class="md-tabs__item">
        <a href="../examples/01_preprocessors_and_utils.html" class="md-tabs__link">
          
  
    
  
  Examples

        </a>
      </li>
    
  

      
        
  
  
  
    <li class="md-tabs__item">
      <a href="../contributing.html" class="md-tabs__link">
        
  
    
  
  Contributing

      </a>
    </li>
  

      
    </ul>
  </div>
</nav>
    
  
</header>
    
    <div class="md-container" data-md-component="container">
      
      
        
      
      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">
          
            
              
              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">
                    


  


  

<nav class="md-nav md-nav--primary md-nav--lifted md-nav--integrated" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../index.html" title="pytorch_widedeep" class="md-nav__button md-logo" aria-label="pytorch_widedeep" data-md-component="logo">
      
  <img src="../assets/images/widedeep_logo.png" alt="logo">

    </a>
    pytorch_widedeep
  </label>
  
    <div class="md-nav__source">
      <a href="https://github.com/jrzaurin/pytorch-widedeep" title="Go to repository" class="md-source" data-md-component="source">
  <div class="md-source__icon md-icon">
    
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M439.55 236.05 244 40.45a28.87 28.87 0 0 0-40.81 0l-40.66 40.63 51.52 51.52c27.06-9.14 52.68 16.77 43.39 43.68l49.66 49.66c34.23-11.8 61.18 31 35.47 56.69-26.49 26.49-70.21-2.87-56-37.34L240.22 199v121.85c25.3 12.54 22.26 41.85 9.08 55a34.34 34.34 0 0 1-48.55 0c-17.57-17.6-11.07-46.91 11.25-56v-123c-20.8-8.51-24.6-30.74-18.64-45L142.57 101 8.45 235.14a28.86 28.86 0 0 0 0 40.81l195.61 195.6a28.86 28.86 0 0 0 40.8 0l194.69-194.69a28.86 28.86 0 0 0 0-40.81"/></svg>
  </div>
  <div class="md-source__repository">
    pytorch_widedeep
  </div>
</a>
    </div>
  
  <ul class="md-nav__list" data-md-scrollfix>
    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../index.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Home
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../installation.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Installation
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../quick_start.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Quick Start
  </span>
  

      </a>
    </li>
  

    
      
      
  
  
    
  
  
  
    
    
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
    
    
      
        
        
      
      
    
    
    <li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
      
        
        
        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_4" checked>
        
          
          <label class="md-nav__link" for="__nav_4" id="__nav_4_label" tabindex="">
            
  
  <span class="md-ellipsis">
    Pytorch-widedeep
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_4_label" aria-expanded="true">
          <label class="md-nav__title" for="__nav_4">
            <span class="md-nav__icon md-icon"></span>
            Pytorch-widedeep
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
                
  
  
  
  
    
    
      
        
          
        
      
        
      
        
      
        
      
        
      
    
    
      
      
    
    
    <li class="md-nav__item md-nav__item--nested">
      
        
        
          
        
        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_4_1" >
        
          
          
          <div class="md-nav__link md-nav__container">
            <a href="utils/index.html" class="md-nav__link ">
              
  
  <span class="md-ellipsis">
    Utils
  </span>
  

            </a>
            
              
              <label class="md-nav__link " for="__nav_4_1" id="__nav_4_1_label" tabindex="0">
                <span class="md-nav__icon md-icon"></span>
              </label>
            
          </div>
        
        <nav class="md-nav" data-md-level="2" aria-labelledby="__nav_4_1_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_4_1">
            <span class="md-nav__icon md-icon"></span>
            Utils
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="utils/deeptabular_utils.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Deeptabular utils
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="utils/fastai_transforms.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Fastai transforms
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="utils/image_utils.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Image utils
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="utils/text_utils.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Text utils
  </span>
  

      </a>
    </li>
  

              
            
          </ul>
        </nav>
      
    </li>
  

              
            
              
                
  
  
    
  
  
  
    <li class="md-nav__item md-nav__item--active">
      
      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
      
      
        
      
      
        <label class="md-nav__link md-nav__link--active" for="__toc">
          
  
  <span class="md-ellipsis">
    Preprocessing
  </span>
  

          <span class="md-nav__icon md-icon"></span>
        </label>
      
      <a href="preprocessing.html" class="md-nav__link md-nav__link--active">
        
  
  <span class="md-ellipsis">
    Preprocessing
  </span>
  

      </a>
      
        

<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
  
  
  
    
  
  
    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      WidePreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      TabPreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="md-nav__link">
    <span class="md-ellipsis">
      Quantizer
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      TextPreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      HFPreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      ImagePreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      DINPreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#chunked-versions" class="md-nav__link">
    <span class="md-ellipsis">
      Chunked versions
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      ChunkWidePreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      ChunkTabPreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      ChunkTextPreprocessor
    </span>
  </a>
  
</li>
      
        <li class="md-nav__item">
  <a href="#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" class="md-nav__link">
    <span class="md-ellipsis">
      ChunkHFPreprocessor
    </span>
  </a>
  
</li>
      
    </ul>
  
</nav>
      
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="load_from_folder.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Load From Folder
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="model_components.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Model Components
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="the_rec_module.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    The Rec Module
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="bayesian_models.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Bayesian models
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="losses.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Losses
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="metrics.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Metrics
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="dataloaders.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Dataloaders
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="callbacks.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Callbacks
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="trainer.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Trainer
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="bayesian_trainer.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Bayesian Trainer
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="self_supervised_pretraining.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Self Supervised Pretraining
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="tab2vec.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Tab2Vec
  </span>
  

      </a>
    </li>
  

              
            
          </ul>
        </nav>
      
    </li>
  

    
      
      
  
  
  
  
    
    
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
        
      
    
    
      
      
    
    
    <li class="md-nav__item md-nav__item--nested">
      
        
        
          
        
        <input class="md-nav__toggle md-toggle md-toggle--indeterminate" type="checkbox" id="__nav_5" >
        
          
          <label class="md-nav__link" for="__nav_5" id="__nav_5_label" tabindex="0">
            
  
  <span class="md-ellipsis">
    Examples
  </span>
  

            <span class="md-nav__icon md-icon"></span>
          </label>
        
        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_5_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_5">
            <span class="md-nav__icon md-icon"></span>
            Examples
          </label>
          <ul class="md-nav__list" data-md-scrollfix>
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/01_preprocessors_and_utils.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    01_preprocessors_and_utils
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/02_model_components.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    02_model_components
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/03_binary_classification_with_defaults.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    03_binary_classification_with_defaults
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/04_regression_with_images_and_text.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    04_regression_with_images_and_text
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/05_save_and_load_model_and_artifacts.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    05_save_and_load_model_and_artifacts
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/06_finetune_and_warmup.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    06_finetune_and_warmup
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/07_custom_components.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    07_custom_components
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/08_custom_dataLoader_imbalanced_dataset.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    08_custom_dataLoader_imbalanced_dataset
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/09_extracting_embeddings.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    09_extracting_embeddings
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/10_3rd_party_integration-RayTune_WnB.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    10_3rd_party_integration-RayTune_WnB
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/11_auc_multiclass.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    11_auc_multiclass
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/12_ZILNLoss_origkeras_vs_pytorch_widedeep.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    12_ZILNLoss_origkeras_vs_pytorch_widedeep
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/13_model_uncertainty_prediction.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    13_model_uncertainty_prediction
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/14_bayesian_models.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    14_bayesian_models
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/15_Self_Supervised_Pretraning_pt1.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    15_Self-Supervised Pre-Training pt 1
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/15_Self_Supervised_Pretraning_pt2.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    15_Self-Supervised Pre-Training pt 2
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/16_Usign_a_custom_hugging_face_model.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    16_Usign-a-custom-hugging-face-model
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/17_feature_importance_via_attention_weights.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    17_feature_importance_via_attention_weights
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/18_wide_and_deep_for_recsys_pt1.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    18_wide_and_deep_for_recsys_pt1
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/18_wide_and_deep_for_recsys_pt2.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    18_wide_and_deep_for_recsys_pt2
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/19_load_from_folder_functionality.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    19_load_from_folder_functionality
  </span>
  

      </a>
    </li>
  

              
            
              
                
  
  
  
  
    <li class="md-nav__item">
      <a href="../examples/20_Using_huggingface_within_widedeep.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    20-Using-huggingface-within-widedeep
  </span>
  

      </a>
    </li>
  

              
            
          </ul>
        </nav>
      
    </li>
  

    
      
      
  
  
  
  
    <li class="md-nav__item">
      <a href="../contributing.html" class="md-nav__link">
        
  
  <span class="md-ellipsis">
    Contributing
  </span>
  

      </a>
    </li>
  

    
  </ul>
</nav>
                  </div>
                </div>
              </div>
            
            
          
          
            <div class="md-content" data-md-component="content">
              <article class="md-content__inner md-typeset">
                
                  

  
  


<h1 id="the-preprocessing-module">The <code>preprocessing</code> module<a class="headerlink" href="#the-preprocessing-module" title="Permanent link">&para;</a></h1>
<p>This module contains the classes that are used to prepare the data before
being passed to the models. There is one Preprocessor per data mode or model
component (<code>wide</code>, <code>deeptabular</code>, <code>deepimage</code> and <code>deeptext</code>) with
the exception of the <code>deeptext</code> component. In this case, two processors are
available: one for the case when no Hugging Face model is used
(<code>TextPreprocessor</code>) and another one when a Hugging Face model is used
(<code>HFPreprocessor</code>).</p>


<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">WidePreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>


        <p>Preprocessor to prepare the wide input dataset</p>
<p>This Preprocessor prepares the data for the wide, linear component.
This linear model is implemented via an Embedding layer that is
connected to the output neuron. <code>WidePreprocessor</code> numerically
encodes all the unique values of all categorical columns <code>wide_cols +
crossed_cols</code>. See the Example below.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>wide_cols</code>
            </td>
            <td>
                  <code><span title="typing.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of strings with the name of the columns that will label
encoded and passed through the <code>wide</code> component</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>crossed_cols</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
and then label encoded. e.g. <em>[('education', 'occupation'), ...]</em>. For
binary features, a cross-product transformation is 1 if and only if
the constituent features are all 1, and 0 otherwise.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></td>
            <td>
                  <code><span title="typing.List">List</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the names of all columns that will be label encoded</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.encoding_dict">encoding_dict</span></code></td>
            <td>
                  <code>Dict</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
column value</code> and the values are the corresponding mapped integer.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></td>
            <td>
                  <code>Dict</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>the inverse encoding dictionary</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.wide_dim">wide_dim</span></code></td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">WidePreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;color&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">],</span> <span class="s1">&#39;size&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;s&#39;</span><span class="p">,</span> <span class="s1">&#39;n&#39;</span><span class="p">,</span> <span class="s1">&#39;l&#39;</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">wide_cols</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;color&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">crossed_cols</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;color&#39;</span><span class="p">,</span> <span class="s1">&#39;size&#39;</span><span class="p">)]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">wide_preprocessor</span> <span class="o">=</span> <span class="n">WidePreprocessor</span><span class="p">(</span><span class="n">wide_cols</span><span class="o">=</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="o">=</span><span class="n">crossed_cols</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_wide</span> <span class="o">=</span> <span class="n">wide_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_wide</span>
<span class="go">array([[1, 4],</span>
<span class="go">       [2, 5],</span>
<span class="go">       [3, 6]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">wide_preprocessor</span><span class="o">.</span><span class="n">encoding_dict</span>
<span class="go">{&#39;color_r&#39;: 1, &#39;color_b&#39;: 2, &#39;color_g&#39;: 3, &#39;color_size_r-s&#39;: 4, &#39;color_size_b-n&#39;: 5, &#39;color_size_g-l&#39;: 6}</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">wide_preprocessor</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">X_wide</span><span class="p">)</span>
<span class="go">  color color_size</span>
<span class="go">0     r        r-s</span>
<span class="go">1     b        b-n</span>
<span class="go">2     g        g-l</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 12</span>
<span class="normal"> 13</span>
<span class="normal"> 14</span>
<span class="normal"> 15</span>
<span class="normal"> 16</span>
<span class="normal"> 17</span>
<span class="normal"> 18</span>
<span class="normal"> 19</span>
<span class="normal"> 20</span>
<span class="normal"> 21</span>
<span class="normal"> 22</span>
<span class="normal"> 23</span>
<span class="normal"> 24</span>
<span class="normal"> 25</span>
<span class="normal"> 26</span>
<span class="normal"> 27</span>
<span class="normal"> 28</span>
<span class="normal"> 29</span>
<span class="normal"> 30</span>
<span class="normal"> 31</span>
<span class="normal"> 32</span>
<span class="normal"> 33</span>
<span class="normal"> 34</span>
<span class="normal"> 35</span>
<span class="normal"> 36</span>
<span class="normal"> 37</span>
<span class="normal"> 38</span>
<span class="normal"> 39</span>
<span class="normal"> 40</span>
<span class="normal"> 41</span>
<span class="normal"> 42</span>
<span class="normal"> 43</span>
<span class="normal"> 44</span>
<span class="normal"> 45</span>
<span class="normal"> 46</span>
<span class="normal"> 47</span>
<span class="normal"> 48</span>
<span class="normal"> 49</span>
<span class="normal"> 50</span>
<span class="normal"> 51</span>
<span class="normal"> 52</span>
<span class="normal"> 53</span>
<span class="normal"> 54</span>
<span class="normal"> 55</span>
<span class="normal"> 56</span>
<span class="normal"> 57</span>
<span class="normal"> 58</span>
<span class="normal"> 59</span>
<span class="normal"> 60</span>
<span class="normal"> 61</span>
<span class="normal"> 62</span>
<span class="normal"> 63</span>
<span class="normal"> 64</span>
<span class="normal"> 65</span>
<span class="normal"> 66</span>
<span class="normal"> 67</span>
<span class="normal"> 68</span>
<span class="normal"> 69</span>
<span class="normal"> 70</span>
<span class="normal"> 71</span>
<span class="normal"> 72</span>
<span class="normal"> 73</span>
<span class="normal"> 74</span>
<span class="normal"> 75</span>
<span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span>
<span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">WidePreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the wide input dataset</span>

<span class="sd">    This Preprocessor prepares the data for the wide, linear component.</span>
<span class="sd">    This linear model is implemented via an Embedding layer that is</span>
<span class="sd">    connected to the output neuron. `WidePreprocessor` numerically</span>
<span class="sd">    encodes all the unique values of all categorical columns `wide_cols +</span>
<span class="sd">    crossed_cols`. See the Example below.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    wide_cols: List</span>
<span class="sd">        List of strings with the name of the columns that will label</span>
<span class="sd">        encoded and passed through the `wide` component</span>
<span class="sd">    crossed_cols: List, default = None</span>
<span class="sd">        List of Tuples with the name of the columns that will be `&#39;crossed&#39;`</span>
<span class="sd">        and then label encoded. e.g. _[(&#39;education&#39;, &#39;occupation&#39;), ...]_. For</span>
<span class="sd">        binary features, a cross-product transformation is 1 if and only if</span>
<span class="sd">        the constituent features are all 1, and 0 otherwise.</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    wide_crossed_cols: List</span>
<span class="sd">        List with the names of all columns that will be label encoded</span>
<span class="sd">    encoding_dict: Dict</span>
<span class="sd">        Dictionary where the keys are the result of pasting `colname + &#39;_&#39; +</span>
<span class="sd">        column value` and the values are the corresponding mapped integer.</span>
<span class="sd">    inverse_encoding_dict: Dict</span>
<span class="sd">        the inverse encoding dictionary</span>
<span class="sd">    wide_dim: int</span>
<span class="sd">        Dimension of the wide model (i.e. dim of the linear layer)</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import WidePreprocessor</span>
<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame({&#39;color&#39;: [&#39;r&#39;, &#39;b&#39;, &#39;g&#39;], &#39;size&#39;: [&#39;s&#39;, &#39;n&#39;, &#39;l&#39;]})</span>
<span class="sd">    &gt;&gt;&gt; wide_cols = [&#39;color&#39;]</span>
<span class="sd">    &gt;&gt;&gt; crossed_cols = [(&#39;color&#39;, &#39;size&#39;)]</span>
<span class="sd">    &gt;&gt;&gt; wide_preprocessor = WidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols)</span>
<span class="sd">    &gt;&gt;&gt; X_wide = wide_preprocessor.fit_transform(df)</span>
<span class="sd">    &gt;&gt;&gt; X_wide</span>
<span class="sd">    array([[1, 4],</span>
<span class="sd">           [2, 5],</span>
<span class="sd">           [3, 6]])</span>
<span class="sd">    &gt;&gt;&gt; wide_preprocessor.encoding_dict</span>
<span class="sd">    {&#39;color_r&#39;: 1, &#39;color_b&#39;: 2, &#39;color_g&#39;: 3, &#39;color_size_r-s&#39;: 4, &#39;color_size_b-n&#39;: 5, &#39;color_size_g-l&#39;: 6}</span>
<span class="sd">    &gt;&gt;&gt; wide_preprocessor.inverse_transform(X_wide)</span>
<span class="sd">      color color_size</span>
<span class="sd">    0     r        r-s</span>
<span class="sd">    1     b        b-n</span>
<span class="sd">    2     g        g-l</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">wide_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">crossed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">WidePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">wide_cols</span> <span class="o">=</span> <span class="n">wide_cols</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="o">=</span> <span class="n">crossed_cols</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;WidePreprocessor&quot;</span><span class="p">:</span>
<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        WidePreprocessor</span>
<span class="sd">            `WidePreprocessor` fitted object</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span> <span class="o">=</span> <span class="n">df_wide</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="n">glob_feature_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span>
            <span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">]</span>
        <span class="p">)</span>
        <span class="c1"># leave 0 for padding/&quot;unseen&quot; categories</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">glob_feature_list</span><span class="p">)}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">wide_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;unseen&quot;</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            transformed input dataframe</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;encoding_dict&quot;</span><span class="p">])</span>
        <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="n">encoded</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">df_wide</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">)])</span>
        <span class="k">for</span> <span class="n">col_i</span><span class="p">,</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">):</span>
            <span class="n">encoded</span><span class="p">[:,</span> <span class="n">col_i</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_wide</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span>
                <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">col</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)]</span>
                    <span class="k">if</span> <span class="n">col</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span>
                    <span class="k">else</span> <span class="mi">0</span>
                <span class="p">)</span>
            <span class="p">)</span>
        <span class="k">return</span> <span class="n">encoded</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;int64&quot;</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
<span class="sd">        return the original values.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        encoded: np.ndarray</span>
<span class="sd">            numpy array with the encoded values that are the output from the</span>
<span class="sd">            `transform` method</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        pd.DataFrame</span>
<span class="sd">            Pandas dataframe with the original values</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">pd</span><span class="o">.</span><span class="n">__version__</span> <span class="o">&gt;=</span> <span class="s2">&quot;2.1.0&quot;</span><span class="p">:</span>
            <span class="n">decoded</span> <span class="o">=</span> <span class="n">decoded</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">x</span><span class="p">])</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">decoded</span> <span class="o">=</span> <span class="n">decoded</span><span class="o">.</span><span class="n">applymap</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">x</span><span class="p">])</span>

        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
            <span class="n">rm_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">col</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="p">])</span>
            <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">rm_str</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">decoded</span>

    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            transformed input dataframe</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_make_global_feature_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
        <span class="n">glob_feature_list</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">column</span> <span class="ow">in</span> <span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
            <span class="n">glob_feature_list</span> <span class="o">+=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_column_feature_list</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="n">column</span><span class="p">])</span>
        <span class="k">return</span> <span class="n">glob_feature_list</span>

    <span class="k">def</span> <span class="nf">_make_column_feature_list</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">s</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">Series</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">:</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">s</span><span class="o">.</span><span class="n">name</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">s</span><span class="o">.</span><span class="n">unique</span><span class="p">()]</span>

    <span class="k">def</span> <span class="nf">_cross_cols</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">):</span>
        <span class="n">df_cc</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="n">crossed_colnames</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">cols</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">cols</span><span class="p">:</span>
                <span class="n">df_cc</span><span class="p">[</span><span class="n">c</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_cc</span><span class="p">[</span><span class="n">c</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;str&quot;</span><span class="p">)</span>
            <span class="n">colname</span> <span class="o">=</span> <span class="s2">&quot;_&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">cols</span><span class="p">)</span>
            <span class="n">df_cc</span><span class="p">[</span><span class="n">colname</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_cc</span><span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">cols</span><span class="p">)]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="s2">&quot;-&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">x</span><span class="p">),</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
            <span class="n">crossed_colnames</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">colname</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">df_cc</span><span class="p">[</span><span class="n">crossed_colnames</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">_prepare_wide</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">df_cc</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cross_cols</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
            <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_cols</span><span class="p">],</span> <span class="n">df_cc</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_cols</span><span class="p">]</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;wide_cols=</span><span class="si">{wide_cols}</span><span class="s2">&quot;</span><span class="p">]</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;crossed_cols=</span><span class="si">{crossed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;WidePreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">









<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Fits the Preprocessor and creates required attributes</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><code>WidePreprocessor</code> fitted object</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;WidePreprocessor&quot;</span><span class="p">:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    WidePreprocessor</span>
<span class="sd">        `WidePreprocessor` fitted object</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span> <span class="o">=</span> <span class="n">df_wide</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
    <span class="n">glob_feature_list</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span>
        <span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">]</span>
    <span class="p">)</span>
    <span class="c1"># leave 0 for padding/&quot;unseen&quot; categories</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">glob_feature_list</span><span class="p">)}</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">wide_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">)</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;unseen&quot;</span>

    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

    <span class="k">return</span> <span class="bp">self</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">transform</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">



<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>transformed input dataframe</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span>
<span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        transformed input dataframe</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;encoding_dict&quot;</span><span class="p">])</span>
    <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
    <span class="n">encoded</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">df_wide</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">)])</span>
    <span class="k">for</span> <span class="n">col_i</span><span class="p">,</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">):</span>
        <span class="n">encoded</span><span class="p">[:,</span> <span class="n">col_i</span><span class="p">]</span> <span class="o">=</span> <span class="n">df_wide</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span>
            <span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">col</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)]</span>
                <span class="k">if</span> <span class="n">col</span> <span class="o">+</span> <span class="s2">&quot;_&quot;</span> <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">x</span><span class="p">)</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span>
                <span class="k">else</span> <span class="mi">0</span>
            <span class="p">)</span>
        <span class="p">)</span>
    <span class="k">return</span> <span class="n">encoded</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;int64&quot;</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">inverse_transform</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Takes as input the output from the <code>transform</code> method and it will
return the original values.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>encoded</code>
            </td>
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>numpy array with the encoded values that are the output from the
<code>transform</code> method</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Pandas dataframe with the original values</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
<span class="sd">    return the original values.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    encoded: np.ndarray</span>
<span class="sd">        numpy array with the encoded values that are the output from the</span>
<span class="sd">        `transform` method</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    pd.DataFrame</span>
<span class="sd">        Pandas dataframe with the original values</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">)</span>

    <span class="k">if</span> <span class="n">pd</span><span class="o">.</span><span class="n">__version__</span> <span class="o">&gt;=</span> <span class="s2">&quot;2.1.0&quot;</span><span class="p">:</span>
        <span class="n">decoded</span> <span class="o">=</span> <span class="n">decoded</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">x</span><span class="p">])</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">decoded</span> <span class="o">=</span> <span class="n">decoded</span><span class="o">.</span><span class="n">applymap</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="n">x</span><span class="p">])</span>

    <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
        <span class="n">rm_str</span> <span class="o">=</span> <span class="s2">&quot;&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="n">col</span><span class="p">,</span> <span class="s2">&quot;_&quot;</span><span class="p">])</span>
        <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="n">rm_str</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
    <span class="k">return</span> <span class="n">decoded</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit_transform</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Combines <code>fit</code> and <code>transform</code></p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>transformed input dataframe</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        transformed input dataframe</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>



  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">TabPreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>


        <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>cat_embed_cols</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List containing the name of the categorical columns that will be
represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
a Tuple with the name and the embedding dimension (e.g.: <em>[
('education',32), ('relationship',16), ...]</em>).<br/> Param alias: <code>embed_cols</code></p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>continuous_cols</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the name of the continuous cols</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>quantization_setup</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If
<code>quantization_setup</code> is an <code>int</code>, all continuous columns will be
quantized using this value as the number of bins. Alternatively, a
dictionary where the keys are the column names to quantize and the
values are the either integers indicating the number of bins or a
list of scalars indicating the bin edges can also be used.<br/>
Param alias: <code>cols_and_bins</code></p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>cols_to_scale</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the names of the columns that will be standarised via
sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in
which case all the continuous cols will be scaled.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>auto_embed_dim</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating whether the embedding dimensions will be
automatically defined via rule of thumb. See <code>embedding_rule</code>
below.</p>
              </div>
            </td>
            <td>
                  <code>True</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>embedding_rule</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Literal">Literal</span>[google, fastai_old, fastai_new]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of
thumb. Choices are:</p>
<ul>
<li>
<p><em>fastai_new</em>: <span class="arithmatex">\(min(600, round(1.6 \times n_{cat}^{0.56}))\)</span></p>
</li>
<li>
<p><em>fastai_old</em>: <span class="arithmatex">\(min(50, (n_{cat}//{2})+1)\)</span></p>
</li>
<li>
<p><em>google</em>: <span class="arithmatex">\(min(600, round(n_{cat}^{0.24}))\)</span></p>
</li>
</ul>
              </div>
            </td>
            <td>
                  <code>&#39;fastai_new&#39;</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>default_embed_dim</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dimension for the embeddings if the embedding dimension is not
provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is
set to <code>False</code>.</p>
              </div>
            </td>
            <td>
                  <code>16</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>with_attention</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating whether the preprocessed data will be passed to an
attention-based model (more precisely a model where all embeddings
must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code>
must just be a list containing just the categorical column names:
e.g.
<em>['education', 'relationship', ...]</em>. This is because they will all be
 encoded using embeddings of the same dim, which will be specified
 later when the model is defined. <br/> Param alias:
 <code>for_transformer</code>, <code>for_matrix_factorization</code>, <code>for_mf</code></p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>with_cls_token</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
when using attention-based models. The final hidden state
corresponding to this token is used as the aggregated representation
for classification and regression tasks. If not, the categorical
and/or continuous embeddings will be concatenated before being passed
to the final MLP (if present).</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>shared_embed</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the embeddings will be "shared" when using
attention-based models. The idea behind <code>shared_embed</code> is
described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
<em>'The goal of having column embedding is to enable the model to
distinguish the classes in one column from those in the other
columns'</em>. In other words, the idea is to let the model learn which
column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>verbose</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>scale</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
 releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
 whether or not to scale/standarise continuous cols. It is important
 to emphasize that all the DL models for tabular data in the library
 also include the possibility of normalising the input continuous
 features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
 <code>scale_cont_cols</code>.</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>already_standard</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
 releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
 name of the continuous cols that do not need to be
 scaled/standarised.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Other Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code>**kwargs</code></td>
            <td>
            </td>
            <td>
              <div class="doc-md-description">
                <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cat_cols">cat_cols</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List containing the names of the categorical columns after processing.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cols_and_bins">cols_and_bins</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary containing the quantization setup after processing if
quantization is requested. This is derived from the
quantization_setup parameter.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.quant_args">quant_args</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.scale_args">scale_args</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary containing arguments passed to StandardScaler.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.label_encoder">label_encoder</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Instance of LabelEncoder used to encode categorical variables.
See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.cat_embed_input">cat_embed_input</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of tuples with the column name, number of individual values for
that column and, if <code>with_attention</code> is set to <code>False</code>, the
corresponding embeddings dim, e.g. <em>[('education', 16, 10),
('relationship', 6, 8), ...]</em>.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.standardize_cols">standardize_cols</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of the columns that will be standardized.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.scaler">scaler</span></code></td>
            <td>
                  <code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization
is requested.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.column_idx">column_idx</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary where keys are column names and values are column indexes.
This is necessary to slice tensors.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.quantizer">quantizer</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>An instance of <code>Quantizer</code> if quantization is requested.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.is_fitted">is_fitted</span></code></td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the preprocessor has been fitted.</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">TabPreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;color&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">],</span> <span class="s1">&#39;size&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;s&#39;</span><span class="p">,</span> <span class="s1">&#39;n&#39;</span><span class="p">,</span> <span class="s1">&#39;l&#39;</span><span class="p">],</span> <span class="s1">&#39;age&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">25</span><span class="p">,</span> <span class="mi">40</span><span class="p">,</span> <span class="mi">55</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;color&#39;</span><span class="p">,</span><span class="mi">5</span><span class="p">),</span> <span class="p">(</span><span class="s1">&#39;size&#39;</span><span class="p">,</span><span class="mi">5</span><span class="p">)]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cont_cols</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;age&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">deep_preprocessor</span> <span class="o">=</span> <span class="n">TabPreprocessor</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="o">=</span><span class="n">cat_embed_cols</span><span class="p">,</span> <span class="n">continuous_cols</span><span class="o">=</span><span class="n">cont_cols</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">deep_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">deep_preprocessor</span><span class="o">.</span><span class="n">cat_embed_cols</span>
<span class="go">[(&#39;color&#39;, 5), (&#39;size&#39;, 5)]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">deep_preprocessor</span><span class="o">.</span><span class="n">column_idx</span>
<span class="go">{&#39;color&#39;: 0, &#39;size&#39;: 1, &#39;age&#39;: 2}</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cont_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;col1&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span> <span class="s2">&quot;col2&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">10</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cont_cols</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;col1&quot;</span><span class="p">,</span> <span class="s2">&quot;col2&quot;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">TabPreprocessor</span><span class="p">(</span><span class="n">continuous_cols</span><span class="o">=</span><span class="n">cont_cols</span><span class="p">,</span> <span class="n">quantization_setup</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ft_cont_df</span> <span class="o">=</span> <span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">cont_df</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># or...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;col1&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mf">0.</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">,</span> <span class="mf">1.</span><span class="p">],</span> <span class="s1">&#39;col2&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mf">1.</span><span class="p">,</span> <span class="mf">1.4</span><span class="p">,</span> <span class="mf">2.</span><span class="p">]}</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tab_preprocessor2</span> <span class="o">=</span> <span class="n">TabPreprocessor</span><span class="p">(</span><span class="n">continuous_cols</span><span class="o">=</span><span class="n">cont_cols</span><span class="p">,</span> <span class="n">quantization_setup</span><span class="o">=</span><span class="n">quantization_setup</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ft_cont_df2</span> <span class="o">=</span> <span class="n">tab_preprocessor2</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">cont_df</span><span class="p">)</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span>
<span class="normal">207</span>
<span class="normal">208</span>
<span class="normal">209</span>
<span class="normal">210</span>
<span class="normal">211</span>
<span class="normal">212</span>
<span class="normal">213</span>
<span class="normal">214</span>
<span class="normal">215</span>
<span class="normal">216</span>
<span class="normal">217</span>
<span class="normal">218</span>
<span class="normal">219</span>
<span class="normal">220</span>
<span class="normal">221</span>
<span class="normal">222</span>
<span class="normal">223</span>
<span class="normal">224</span>
<span class="normal">225</span>
<span class="normal">226</span>
<span class="normal">227</span>
<span class="normal">228</span>
<span class="normal">229</span>
<span class="normal">230</span>
<span class="normal">231</span>
<span class="normal">232</span>
<span class="normal">233</span>
<span class="normal">234</span>
<span class="normal">235</span>
<span class="normal">236</span>
<span class="normal">237</span>
<span class="normal">238</span>
<span class="normal">239</span>
<span class="normal">240</span>
<span class="normal">241</span>
<span class="normal">242</span>
<span class="normal">243</span>
<span class="normal">244</span>
<span class="normal">245</span>
<span class="normal">246</span>
<span class="normal">247</span>
<span class="normal">248</span>
<span class="normal">249</span>
<span class="normal">250</span>
<span class="normal">251</span>
<span class="normal">252</span>
<span class="normal">253</span>
<span class="normal">254</span>
<span class="normal">255</span>
<span class="normal">256</span>
<span class="normal">257</span>
<span class="normal">258</span>
<span class="normal">259</span>
<span class="normal">260</span>
<span class="normal">261</span>
<span class="normal">262</span>
<span class="normal">263</span>
<span class="normal">264</span>
<span class="normal">265</span>
<span class="normal">266</span>
<span class="normal">267</span>
<span class="normal">268</span>
<span class="normal">269</span>
<span class="normal">270</span>
<span class="normal">271</span>
<span class="normal">272</span>
<span class="normal">273</span>
<span class="normal">274</span>
<span class="normal">275</span>
<span class="normal">276</span>
<span class="normal">277</span>
<span class="normal">278</span>
<span class="normal">279</span>
<span class="normal">280</span>
<span class="normal">281</span>
<span class="normal">282</span>
<span class="normal">283</span>
<span class="normal">284</span>
<span class="normal">285</span>
<span class="normal">286</span>
<span class="normal">287</span>
<span class="normal">288</span>
<span class="normal">289</span>
<span class="normal">290</span>
<span class="normal">291</span>
<span class="normal">292</span>
<span class="normal">293</span>
<span class="normal">294</span>
<span class="normal">295</span>
<span class="normal">296</span>
<span class="normal">297</span>
<span class="normal">298</span>
<span class="normal">299</span>
<span class="normal">300</span>
<span class="normal">301</span>
<span class="normal">302</span>
<span class="normal">303</span>
<span class="normal">304</span>
<span class="normal">305</span>
<span class="normal">306</span>
<span class="normal">307</span>
<span class="normal">308</span>
<span class="normal">309</span>
<span class="normal">310</span>
<span class="normal">311</span>
<span class="normal">312</span>
<span class="normal">313</span>
<span class="normal">314</span>
<span class="normal">315</span>
<span class="normal">316</span>
<span class="normal">317</span>
<span class="normal">318</span>
<span class="normal">319</span>
<span class="normal">320</span>
<span class="normal">321</span>
<span class="normal">322</span>
<span class="normal">323</span>
<span class="normal">324</span>
<span class="normal">325</span>
<span class="normal">326</span>
<span class="normal">327</span>
<span class="normal">328</span>
<span class="normal">329</span>
<span class="normal">330</span>
<span class="normal">331</span>
<span class="normal">332</span>
<span class="normal">333</span>
<span class="normal">334</span>
<span class="normal">335</span>
<span class="normal">336</span>
<span class="normal">337</span>
<span class="normal">338</span>
<span class="normal">339</span>
<span class="normal">340</span>
<span class="normal">341</span>
<span class="normal">342</span>
<span class="normal">343</span>
<span class="normal">344</span>
<span class="normal">345</span>
<span class="normal">346</span>
<span class="normal">347</span>
<span class="normal">348</span>
<span class="normal">349</span>
<span class="normal">350</span>
<span class="normal">351</span>
<span class="normal">352</span>
<span class="normal">353</span>
<span class="normal">354</span>
<span class="normal">355</span>
<span class="normal">356</span>
<span class="normal">357</span>
<span class="normal">358</span>
<span class="normal">359</span>
<span class="normal">360</span>
<span class="normal">361</span>
<span class="normal">362</span>
<span class="normal">363</span>
<span class="normal">364</span>
<span class="normal">365</span>
<span class="normal">366</span>
<span class="normal">367</span>
<span class="normal">368</span>
<span class="normal">369</span>
<span class="normal">370</span>
<span class="normal">371</span>
<span class="normal">372</span>
<span class="normal">373</span>
<span class="normal">374</span>
<span class="normal">375</span>
<span class="normal">376</span>
<span class="normal">377</span>
<span class="normal">378</span>
<span class="normal">379</span>
<span class="normal">380</span>
<span class="normal">381</span>
<span class="normal">382</span>
<span class="normal">383</span>
<span class="normal">384</span>
<span class="normal">385</span>
<span class="normal">386</span>
<span class="normal">387</span>
<span class="normal">388</span>
<span class="normal">389</span>
<span class="normal">390</span>
<span class="normal">391</span>
<span class="normal">392</span>
<span class="normal">393</span>
<span class="normal">394</span>
<span class="normal">395</span>
<span class="normal">396</span>
<span class="normal">397</span>
<span class="normal">398</span>
<span class="normal">399</span>
<span class="normal">400</span>
<span class="normal">401</span>
<span class="normal">402</span>
<span class="normal">403</span>
<span class="normal">404</span>
<span class="normal">405</span>
<span class="normal">406</span>
<span class="normal">407</span>
<span class="normal">408</span>
<span class="normal">409</span>
<span class="normal">410</span>
<span class="normal">411</span>
<span class="normal">412</span>
<span class="normal">413</span>
<span class="normal">414</span>
<span class="normal">415</span>
<span class="normal">416</span>
<span class="normal">417</span>
<span class="normal">418</span>
<span class="normal">419</span>
<span class="normal">420</span>
<span class="normal">421</span>
<span class="normal">422</span>
<span class="normal">423</span>
<span class="normal">424</span>
<span class="normal">425</span>
<span class="normal">426</span>
<span class="normal">427</span>
<span class="normal">428</span>
<span class="normal">429</span>
<span class="normal">430</span>
<span class="normal">431</span>
<span class="normal">432</span>
<span class="normal">433</span>
<span class="normal">434</span>
<span class="normal">435</span>
<span class="normal">436</span>
<span class="normal">437</span>
<span class="normal">438</span>
<span class="normal">439</span>
<span class="normal">440</span>
<span class="normal">441</span>
<span class="normal">442</span>
<span class="normal">443</span>
<span class="normal">444</span>
<span class="normal">445</span>
<span class="normal">446</span>
<span class="normal">447</span>
<span class="normal">448</span>
<span class="normal">449</span>
<span class="normal">450</span>
<span class="normal">451</span>
<span class="normal">452</span>
<span class="normal">453</span>
<span class="normal">454</span>
<span class="normal">455</span>
<span class="normal">456</span>
<span class="normal">457</span>
<span class="normal">458</span>
<span class="normal">459</span>
<span class="normal">460</span>
<span class="normal">461</span>
<span class="normal">462</span>
<span class="normal">463</span>
<span class="normal">464</span>
<span class="normal">465</span>
<span class="normal">466</span>
<span class="normal">467</span>
<span class="normal">468</span>
<span class="normal">469</span>
<span class="normal">470</span>
<span class="normal">471</span>
<span class="normal">472</span>
<span class="normal">473</span>
<span class="normal">474</span>
<span class="normal">475</span>
<span class="normal">476</span>
<span class="normal">477</span>
<span class="normal">478</span>
<span class="normal">479</span>
<span class="normal">480</span>
<span class="normal">481</span>
<span class="normal">482</span>
<span class="normal">483</span>
<span class="normal">484</span>
<span class="normal">485</span>
<span class="normal">486</span>
<span class="normal">487</span>
<span class="normal">488</span>
<span class="normal">489</span>
<span class="normal">490</span>
<span class="normal">491</span>
<span class="normal">492</span>
<span class="normal">493</span>
<span class="normal">494</span>
<span class="normal">495</span>
<span class="normal">496</span>
<span class="normal">497</span>
<span class="normal">498</span>
<span class="normal">499</span>
<span class="normal">500</span>
<span class="normal">501</span>
<span class="normal">502</span>
<span class="normal">503</span>
<span class="normal">504</span>
<span class="normal">505</span>
<span class="normal">506</span>
<span class="normal">507</span>
<span class="normal">508</span>
<span class="normal">509</span>
<span class="normal">510</span>
<span class="normal">511</span>
<span class="normal">512</span>
<span class="normal">513</span>
<span class="normal">514</span>
<span class="normal">515</span>
<span class="normal">516</span>
<span class="normal">517</span>
<span class="normal">518</span>
<span class="normal">519</span>
<span class="normal">520</span>
<span class="normal">521</span>
<span class="normal">522</span>
<span class="normal">523</span>
<span class="normal">524</span>
<span class="normal">525</span>
<span class="normal">526</span>
<span class="normal">527</span>
<span class="normal">528</span>
<span class="normal">529</span>
<span class="normal">530</span>
<span class="normal">531</span>
<span class="normal">532</span>
<span class="normal">533</span>
<span class="normal">534</span>
<span class="normal">535</span>
<span class="normal">536</span>
<span class="normal">537</span>
<span class="normal">538</span>
<span class="normal">539</span>
<span class="normal">540</span>
<span class="normal">541</span>
<span class="normal">542</span>
<span class="normal">543</span>
<span class="normal">544</span>
<span class="normal">545</span>
<span class="normal">546</span>
<span class="normal">547</span>
<span class="normal">548</span>
<span class="normal">549</span>
<span class="normal">550</span>
<span class="normal">551</span>
<span class="normal">552</span>
<span class="normal">553</span>
<span class="normal">554</span>
<span class="normal">555</span>
<span class="normal">556</span>
<span class="normal">557</span>
<span class="normal">558</span>
<span class="normal">559</span>
<span class="normal">560</span>
<span class="normal">561</span>
<span class="normal">562</span>
<span class="normal">563</span>
<span class="normal">564</span>
<span class="normal">565</span>
<span class="normal">566</span>
<span class="normal">567</span>
<span class="normal">568</span>
<span class="normal">569</span>
<span class="normal">570</span>
<span class="normal">571</span>
<span class="normal">572</span>
<span class="normal">573</span>
<span class="normal">574</span>
<span class="normal">575</span>
<span class="normal">576</span>
<span class="normal">577</span>
<span class="normal">578</span>
<span class="normal">579</span>
<span class="normal">580</span>
<span class="normal">581</span>
<span class="normal">582</span>
<span class="normal">583</span>
<span class="normal">584</span>
<span class="normal">585</span>
<span class="normal">586</span>
<span class="normal">587</span>
<span class="normal">588</span>
<span class="normal">589</span>
<span class="normal">590</span>
<span class="normal">591</span>
<span class="normal">592</span>
<span class="normal">593</span>
<span class="normal">594</span>
<span class="normal">595</span>
<span class="normal">596</span>
<span class="normal">597</span>
<span class="normal">598</span>
<span class="normal">599</span>
<span class="normal">600</span>
<span class="normal">601</span>
<span class="normal">602</span>
<span class="normal">603</span>
<span class="normal">604</span>
<span class="normal">605</span>
<span class="normal">606</span>
<span class="normal">607</span>
<span class="normal">608</span>
<span class="normal">609</span>
<span class="normal">610</span>
<span class="normal">611</span>
<span class="normal">612</span>
<span class="normal">613</span>
<span class="normal">614</span>
<span class="normal">615</span>
<span class="normal">616</span>
<span class="normal">617</span>
<span class="normal">618</span>
<span class="normal">619</span>
<span class="normal">620</span>
<span class="normal">621</span>
<span class="normal">622</span>
<span class="normal">623</span>
<span class="normal">624</span>
<span class="normal">625</span>
<span class="normal">626</span>
<span class="normal">627</span>
<span class="normal">628</span>
<span class="normal">629</span>
<span class="normal">630</span>
<span class="normal">631</span>
<span class="normal">632</span>
<span class="normal">633</span>
<span class="normal">634</span>
<span class="normal">635</span>
<span class="normal">636</span>
<span class="normal">637</span>
<span class="normal">638</span>
<span class="normal">639</span>
<span class="normal">640</span>
<span class="normal">641</span>
<span class="normal">642</span>
<span class="normal">643</span>
<span class="normal">644</span>
<span class="normal">645</span>
<span class="normal">646</span>
<span class="normal">647</span>
<span class="normal">648</span>
<span class="normal">649</span>
<span class="normal">650</span>
<span class="normal">651</span>
<span class="normal">652</span>
<span class="normal">653</span>
<span class="normal">654</span>
<span class="normal">655</span>
<span class="normal">656</span>
<span class="normal">657</span>
<span class="normal">658</span>
<span class="normal">659</span>
<span class="normal">660</span>
<span class="normal">661</span>
<span class="normal">662</span>
<span class="normal">663</span>
<span class="normal">664</span>
<span class="normal">665</span>
<span class="normal">666</span>
<span class="normal">667</span>
<span class="normal">668</span>
<span class="normal">669</span>
<span class="normal">670</span>
<span class="normal">671</span>
<span class="normal">672</span>
<span class="normal">673</span>
<span class="normal">674</span>
<span class="normal">675</span>
<span class="normal">676</span>
<span class="normal">677</span>
<span class="normal">678</span>
<span class="normal">679</span>
<span class="normal">680</span>
<span class="normal">681</span>
<span class="normal">682</span>
<span class="normal">683</span>
<span class="normal">684</span>
<span class="normal">685</span>
<span class="normal">686</span>
<span class="normal">687</span>
<span class="normal">688</span>
<span class="normal">689</span>
<span class="normal">690</span>
<span class="normal">691</span>
<span class="normal">692</span>
<span class="normal">693</span>
<span class="normal">694</span>
<span class="normal">695</span>
<span class="normal">696</span>
<span class="normal">697</span>
<span class="normal">698</span>
<span class="normal">699</span>
<span class="normal">700</span>
<span class="normal">701</span>
<span class="normal">702</span>
<span class="normal">703</span>
<span class="normal">704</span>
<span class="normal">705</span>
<span class="normal">706</span>
<span class="normal">707</span>
<span class="normal">708</span>
<span class="normal">709</span>
<span class="normal">710</span>
<span class="normal">711</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TabPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the `deeptabular` component input dataset</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    cat_embed_cols: List, default = None</span>
<span class="sd">        List containing the name of the categorical columns that will be</span>
<span class="sd">        represented by embeddings (e.g. _[&#39;education&#39;, &#39;relationship&#39;, ...]_) or</span>
<span class="sd">        a Tuple with the name and the embedding dimension (e.g.: _[</span>
<span class="sd">        (&#39;education&#39;,32), (&#39;relationship&#39;,16), ...]_).&lt;br/&gt; Param alias: `embed_cols`</span>
<span class="sd">    continuous_cols: List, default = None</span>
<span class="sd">        List with the name of the continuous cols</span>
<span class="sd">    quantization_setup: int or Dict, default = None</span>
<span class="sd">        Continuous columns can be turned into categorical via `pd.cut`. If</span>
<span class="sd">        `quantization_setup` is an `int`, all continuous columns will be</span>
<span class="sd">        quantized using this value as the number of bins. Alternatively, a</span>
<span class="sd">        dictionary where the keys are the column names to quantize and the</span>
<span class="sd">        values are the either integers indicating the number of bins or a</span>
<span class="sd">        list of scalars indicating the bin edges can also be used.&lt;br/&gt;</span>
<span class="sd">        Param alias: `cols_and_bins`</span>
<span class="sd">    cols_to_scale: List or str, default = None,</span>
<span class="sd">        List with the names of the columns that will be standarised via</span>
<span class="sd">        sklearn&#39;s `StandardScaler`. It can also be the string `&#39;all&#39;` in</span>
<span class="sd">        which case all the continuous cols will be scaled.</span>
<span class="sd">    auto_embed_dim: bool, default = True</span>
<span class="sd">        Boolean indicating whether the embedding dimensions will be</span>
<span class="sd">        automatically defined via rule of thumb. See `embedding_rule`</span>
<span class="sd">        below.</span>
<span class="sd">    embedding_rule: str, default = &#39;fastai_new&#39;</span>
<span class="sd">        If `auto_embed_dim=True`, this is the choice of embedding rule of</span>
<span class="sd">        thumb. Choices are:</span>

<span class="sd">        - _fastai_new_: $min(600, round(1.6 \times n_{cat}^{0.56}))$</span>

<span class="sd">        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$</span>

<span class="sd">        - _google_: $min(600, round(n_{cat}^{0.24}))$</span>
<span class="sd">    default_embed_dim: int, default=16</span>
<span class="sd">        Dimension for the embeddings if the embedding dimension is not</span>
<span class="sd">        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is</span>
<span class="sd">        set to `False`.</span>
<span class="sd">    with_attention: bool, default = False</span>
<span class="sd">        Boolean indicating whether the preprocessed data will be passed to an</span>
<span class="sd">        attention-based model (more precisely a model where all embeddings</span>
<span class="sd">        must have the same dimensions). If `True`, the param `cat_embed_cols`</span>
<span class="sd">        must just be a list containing just the categorical column names:</span>
<span class="sd">        e.g.</span>
<span class="sd">        _[&#39;education&#39;, &#39;relationship&#39;, ...]_. This is because they will all be</span>
<span class="sd">         encoded using embeddings of the same dim, which will be specified</span>
<span class="sd">         later when the model is defined. &lt;br/&gt; Param alias:</span>
<span class="sd">         `for_transformer`, `for_matrix_factorization`, `for_mf`</span>
<span class="sd">    with_cls_token: bool, default = False</span>
<span class="sd">        Boolean indicating if a `&#39;[CLS]&#39;` token will be added to the dataset</span>
<span class="sd">        when using attention-based models. The final hidden state</span>
<span class="sd">        corresponding to this token is used as the aggregated representation</span>
<span class="sd">        for classification and regression tasks. If not, the categorical</span>
<span class="sd">        and/or continuous embeddings will be concatenated before being passed</span>
<span class="sd">        to the final MLP (if present).</span>
<span class="sd">    shared_embed: bool, default = False</span>
<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot; when using</span>
<span class="sd">        attention-based models. The idea behind `shared_embed` is</span>
<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
<span class="sd">        distinguish the classes in one column from those in the other</span>
<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
<span class="sd">    verbose: int, default = 1</span>
<span class="sd">    scale: bool, default = False</span>
<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating</span>
<span class="sd">         whether or not to scale/standarise continuous cols. It is important</span>
<span class="sd">         to emphasize that all the DL models for tabular data in the library</span>
<span class="sd">         also include the possibility of normalising the input continuous</span>
<span class="sd">         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:</span>
<span class="sd">         `scale_cont_cols`.</span>
<span class="sd">    already_standard: List, default = None</span>
<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the</span>
<span class="sd">         name of the continuous cols that do not need to be</span>
<span class="sd">         scaled/standarised.</span>

<span class="sd">    Other Parameters</span>
<span class="sd">    ----------------</span>
<span class="sd">    **kwargs: dict</span>
<span class="sd">        `pd.cut` and `StandardScaler` related args</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    cat_cols: List[str]</span>
<span class="sd">        List containing the names of the categorical columns after processing.</span>
<span class="sd">    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]</span>
<span class="sd">        Dictionary containing the quantization setup after processing if</span>
<span class="sd">        quantization is requested. This is derived from the</span>
<span class="sd">        quantization_setup parameter.</span>
<span class="sd">    quant_args: Dict</span>
<span class="sd">        Dictionary containing arguments passed to pandas.cut() function for quantization.</span>
<span class="sd">    scale_args: Dict</span>
<span class="sd">        Dictionary containing arguments passed to StandardScaler.</span>
<span class="sd">    label_encoder: LabelEncoder</span>
<span class="sd">        Instance of LabelEncoder used to encode categorical variables.</span>
<span class="sd">        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.</span>
<span class="sd">    cat_embed_input: List</span>
<span class="sd">        List of tuples with the column name, number of individual values for</span>
<span class="sd">        that column and, if `with_attention` is set to `False`, the</span>
<span class="sd">        corresponding embeddings dim, e.g. _[(&#39;education&#39;, 16, 10),</span>
<span class="sd">        (&#39;relationship&#39;, 6, 8), ...]_.</span>
<span class="sd">    standardize_cols: List</span>
<span class="sd">        List of the columns that will be standardized.</span>
<span class="sd">    scaler: StandardScaler</span>
<span class="sd">        An instance of `sklearn.preprocessing.StandardScaler` if standardization</span>
<span class="sd">        is requested.</span>
<span class="sd">    column_idx: Dict</span>
<span class="sd">        Dictionary where keys are column names and values are column indexes.</span>
<span class="sd">        This is necessary to slice tensors.</span>
<span class="sd">    quantizer: Quantizer</span>
<span class="sd">        An instance of `Quantizer` if quantization is requested.</span>
<span class="sd">    is_fitted: bool</span>
<span class="sd">        Boolean indicating if the preprocessor has been fitted.</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; import numpy as np</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TabPreprocessor</span>
<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame({&#39;color&#39;: [&#39;r&#39;, &#39;b&#39;, &#39;g&#39;], &#39;size&#39;: [&#39;s&#39;, &#39;n&#39;, &#39;l&#39;], &#39;age&#39;: [25, 40, 55]})</span>
<span class="sd">    &gt;&gt;&gt; cat_embed_cols = [(&#39;color&#39;,5), (&#39;size&#39;,5)]</span>
<span class="sd">    &gt;&gt;&gt; cont_cols = [&#39;age&#39;]</span>
<span class="sd">    &gt;&gt;&gt; deep_preprocessor = TabPreprocessor(cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols)</span>
<span class="sd">    &gt;&gt;&gt; X_tab = deep_preprocessor.fit_transform(df)</span>
<span class="sd">    &gt;&gt;&gt; deep_preprocessor.cat_embed_cols</span>
<span class="sd">    [(&#39;color&#39;, 5), (&#39;size&#39;, 5)]</span>
<span class="sd">    &gt;&gt;&gt; deep_preprocessor.column_idx</span>
<span class="sd">    {&#39;color&#39;: 0, &#39;size&#39;: 1, &#39;age&#39;: 2}</span>
<span class="sd">    &gt;&gt;&gt; cont_df = pd.DataFrame({&quot;col1&quot;: np.random.rand(10), &quot;col2&quot;: np.random.rand(10) + 1})</span>
<span class="sd">    &gt;&gt;&gt; cont_cols = [&quot;col1&quot;, &quot;col2&quot;]</span>
<span class="sd">    &gt;&gt;&gt; tab_preprocessor = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=3)</span>
<span class="sd">    &gt;&gt;&gt; ft_cont_df = tab_preprocessor.fit_transform(cont_df)</span>
<span class="sd">    &gt;&gt;&gt; # or...</span>
<span class="sd">    &gt;&gt;&gt; quantization_setup = {&#39;col1&#39;: [0., 0.4, 1.], &#39;col2&#39;: [1., 1.4, 2.]}</span>
<span class="sd">    &gt;&gt;&gt; tab_preprocessor2 = TabPreprocessor(continuous_cols=cont_cols, quantization_setup=quantization_setup)</span>
<span class="sd">    &gt;&gt;&gt; ft_cont_df2 = tab_preprocessor2.fit_transform(cont_df)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">,</span> <span class="s2">&quot;for_matrix_factorization&quot;</span><span class="p">,</span> <span class="s2">&quot;for_mf&quot;</span><span class="p">])</span>
    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;embed_cols&quot;</span><span class="p">])</span>
    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;scale&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;scale_cont_cols&quot;</span><span class="p">])</span>
    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;quantization_setup&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;cols_and_bins&quot;</span><span class="p">])</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
            <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">auto_embed_dim</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
        <span class="n">embedding_rule</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_old&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
        <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
        <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">with_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
        <span class="o">*</span><span class="p">,</span>
        <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">TabPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">=</span> <span class="n">cols_to_scale</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="o">=</span> <span class="n">already_standard</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span> <span class="o">=</span> <span class="n">auto_embed_dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">=</span> <span class="n">embedding_rule</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">=</span> <span class="n">default_embed_dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="o">=</span> <span class="n">with_attention</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="o">=</span> <span class="n">with_cls_token</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span> <span class="o">=</span> <span class="n">shared_embed</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">pd</span><span class="o">.</span><span class="n">cut</span><span class="o">.</span><span class="vm">__code__</span><span class="o">.</span><span class="n">co_varnames</span>
        <span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">kwargs</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">StandardScaler</span><span class="p">()</span><span class="o">.</span><span class="n">get_params</span><span class="p">()</span>
        <span class="p">}</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_check_inputs</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">with_cls_token</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="p">(</span>
                <span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span> <span class="o">+</span> <span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[operator]</span>
                <span class="k">if</span> <span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
                <span class="k">else</span> <span class="p">[</span><span class="s2">&quot;cls_token&quot;</span><span class="p">]</span>
            <span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[assignment]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        TabPreprocessor</span>
<span class="sd">            `TabPreprocessor` fitted object</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>

        <span class="c1"># Categorical embeddings logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">(</span>
                <span class="p">[]</span>
            <span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">df_cat</span><span class="p">,</span> <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_categorical</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
                <span class="n">columns_to_encode</span><span class="o">=</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span>
                <span class="n">shared_embed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">,</span>
                <span class="n">with_attention</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">,</span>
            <span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>

            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)))</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="n">cat_embed_dim</span><span class="p">[</span><span class="n">k</span><span class="p">]))</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="p">)})</span>

        <span class="c1"># Continuous columns logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">df_cont</span><span class="p">,</span> <span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_continuous</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>

            <span class="c1"># Standardization logic</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
                    <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
                <span class="p">)</span>
            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Continuous columns will not be normalised&quot;</span><span class="p">)</span>

            <span class="c1"># Quantization logic</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="c1"># we do not run &#39;Quantizer.fit&#39; here since in the wild case</span>
                <span class="c1"># someone wants standardization and quantization for the same</span>
                <span class="c1"># columns, the Quantizer will run on the scaled data</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>

                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
                    <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">cont_embed_dim</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">))</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">cont_embed_dim</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
                <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)}</span>
            <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the processed `dataframe` as a np.ndarray</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            transformed input dataframe</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">)</span>

        <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">df_cat</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">]</span>
            <span class="n">df_cat</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">df_cont</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
            <span class="c1"># Standardization logic</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">:</span>
                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span>
                    <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
                <span class="p">)</span>
            <span class="c1"># Quantization logic</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="c1"># Adjustment so I don&#39;t have to override the method</span>
                <span class="c1"># in &#39;ChunkTabPreprocessor&#39;</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
                    <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df_cat</span><span class="p">,</span> <span class="n">df_cont</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cat</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
            <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
                <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cont</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

        <span class="k">return</span> <span class="n">df_deep</span><span class="o">.</span><span class="n">values</span>

    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="s2">&quot;float&quot;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
<span class="sd">        return the original values.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        encoded: np.ndarray</span>
<span class="sd">            array with the output of the `transform` method</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        pd.DataFrame</span>
<span class="sd">            Pandas dataframe with the original values</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
        <span class="c1"># embeddings back to original category</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">decoded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">decoded</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="c1"># quantized cols to the mid point</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
                    <span class="nb">print</span><span class="p">(</span>
                        <span class="s2">&quot;Note that quantized cols will be turned into the mid point of &quot;</span>
                        <span class="s2">&quot;the corresponding bin&quot;</span>
                    <span class="p">)</span>
                <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">inversed_bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                    <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
            <span class="c1"># continuous_cols back to non-standarised</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span>
                    <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span>
                <span class="p">)</span>
            <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>  <span class="c1"># KeyError:</span>
                <span class="k">pass</span>

        <span class="k">if</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
            <span class="n">decoded</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;cls_token&quot;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

        <span class="k">return</span> <span class="n">decoded</span>

    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            transformed input dataframe</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_insert_cls_token</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
        <span class="n">df_cls</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="n">df_cls</span><span class="o">.</span><span class="n">insert</span><span class="p">(</span><span class="n">loc</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">column</span><span class="o">=</span><span class="s2">&quot;cls_token&quot;</span><span class="p">,</span> <span class="n">value</span><span class="o">=</span><span class="s2">&quot;[CLS]&quot;</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">df_cls</span>

    <span class="k">def</span> <span class="nf">_prepare_categorical</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]:</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">emb</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">emb</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">]</span>
            <span class="n">cat_embed_dim</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[assignment]</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">:</span>
                <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">str</span><span class="p">),</span> <span class="p">(</span>
                    <span class="s2">&quot;If &#39;auto_embed_dim&#39; is &#39;True&#39; and &#39;with_attention&#39; is &#39;False&#39;, &quot;</span>
                    <span class="s2">&quot;&#39;cat_embed_cols&#39; must be a list of strings with the columns to &quot;</span>
                    <span class="s2">&quot;be encoded as embeddings.&quot;</span>
                <span class="p">)</span>
                <span class="n">n_cats</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span><span class="n">col</span><span class="p">:</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">nunique</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">}</span>  <span class="c1"># type: ignore[misc]</span>
                <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="p">{</span>
                    <span class="n">col</span><span class="p">:</span> <span class="n">embed_sz_rule</span><span class="p">(</span><span class="n">n_cat</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">)</span>  <span class="c1"># type: ignore[misc]</span>
                    <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span> <span class="ow">in</span> <span class="n">n_cats</span><span class="o">.</span><span class="n">items</span><span class="p">()</span>
                <span class="p">}</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="p">{</span>
                    <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[misc]</span>
                <span class="p">}</span>  <span class="c1"># type: ignore</span>
        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">],</span> <span class="n">cat_embed_dim</span>

    <span class="k">def</span> <span class="nf">_prepare_continuous</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]:</span>
        <span class="c1"># Standardization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">!=</span> <span class="s2">&quot;all&quot;</span>
                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
            <span class="p">)</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">[</span>
                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span>
                <span class="p">]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="c1"># Quantization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="c1"># the quantized columns are then treated as categorical</span>
            <span class="n">quant_cont_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span> <span class="o">=</span> <span class="p">{}</span>
                <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span>
                    <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                        <span class="p">(</span>
                            <span class="n">col</span><span class="p">,</span>
                            <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span>
                            <span class="p">(</span>
                                <span class="n">embed_sz_rule</span><span class="p">(</span>
                                    <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span>  <span class="c1"># type: ignore[arg-type]</span>
                                <span class="p">)</span>
                                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span>
                                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span>
                            <span class="p">),</span>
                        <span class="p">)</span>
                    <span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">val</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
                        <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                            <span class="p">(</span>
                                <span class="n">col</span><span class="p">,</span>
                                <span class="n">val</span><span class="p">,</span>
                                <span class="p">(</span>
                                    <span class="n">embed_sz_rule</span><span class="p">(</span>
                                        <span class="n">val</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span>  <span class="c1"># type: ignore[arg-type]</span>
                                    <span class="p">)</span>
                                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span>
                                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span>
                                <span class="p">),</span>
                            <span class="p">)</span>
                        <span class="p">)</span>
                    <span class="k">else</span><span class="p">:</span>
                        <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                            <span class="p">(</span>
                                <span class="n">col</span><span class="p">,</span>
                                <span class="nb">len</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
                                <span class="p">(</span>
                                    <span class="n">embed_sz_rule</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">val</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">)</span>  <span class="c1"># type: ignore[arg-type]</span>
                                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span>
                                    <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span>
                                <span class="p">),</span>
                            <span class="p">)</span>
                        <span class="p">)</span>

                <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="o">=</span> <span class="kc">None</span>
            <span class="n">quant_cont_embed_input</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">],</span> <span class="n">quant_cont_embed_input</span>

    <span class="k">def</span> <span class="nf">_check_inputs</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">cat_embed_cols</span><span class="p">):</span>  <span class="c1"># noqa: C901</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
                <span class="s2">&quot;&#39;scale&#39; and &#39;already_standard&#39; will be deprecated in the next release. &quot;</span>
                <span class="s2">&quot;Please use &#39;cols_to_scale&#39; instead&quot;</span><span class="p">,</span>
                <span class="ne">DeprecationWarning</span><span class="p">,</span>
                <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
            <span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">[</span>
                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span>
                <span class="p">]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">standardize_cols</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="k">if</span> <span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
                <span class="n">cols_to_quantize_and_standardize</span> <span class="o">=</span> <span class="p">[</span>
                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">standardize_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
                <span class="p">]</span>
            <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
                <span class="n">cols_to_quantize_and_standardize</span> <span class="o">=</span> <span class="p">[</span>
                    <span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">standardize_cols</span> <span class="k">if</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span>
                <span class="p">]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">cols_to_quantize_and_standardize</span> <span class="o">=</span> <span class="kc">None</span>
            <span class="k">if</span> <span class="n">cols_to_quantize_and_standardize</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
                    <span class="sa">f</span><span class="s2">&quot;the following columns: </span><span class="si">{</span><span class="n">cols_to_quantize_and_standardize</span><span class="si">}</span><span class="s2"> will be first scaled&quot;</span>
                    <span class="s2">&quot; using a StandardScaler and then quantized. Make sure this is what you really want&quot;</span>
                <span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="ow">and</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
                <span class="s2">&quot;If &#39;with_cls_token&#39; is set to &#39;True&#39;, &#39;with_attention&#39; will be automatically &quot;</span>
                <span class="s2">&quot;to &#39;True&#39; if is &#39;False&#39;&quot;</span><span class="p">,</span>
                <span class="ne">UserWarning</span><span class="p">,</span>
            <span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">if</span> <span class="p">(</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">)</span> <span class="ow">and</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;&#39;cat_embed_cols&#39; and &#39;continuous_cols&#39; are &#39;None&#39;. Please, define at least one of the two.&quot;</span>
            <span class="p">)</span>

        <span class="k">if</span> <span class="p">(</span>
            <span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
            <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
            <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">))</span> <span class="o">&gt;</span> <span class="mi">0</span>
        <span class="p">):</span>
            <span class="n">overlapping_cols</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span>
                <span class="n">np</span><span class="o">.</span><span class="n">intersect1d</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
            <span class="p">)</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;Currently passing columns as both categorical and continuum is not supported.&quot;</span>
                <span class="s2">&quot; Please, choose one or the other for the following columns: </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                    <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">overlapping_cols</span><span class="p">)</span>
                <span class="p">)</span>
            <span class="p">)</span>
        <span class="n">transformer_error_message</span> <span class="o">=</span> <span class="p">(</span>
            <span class="s2">&quot;If with_attention is &#39;True&#39; cat_embed_cols must be a list &quot;</span>
            <span class="s2">&quot; of strings with the columns to be encoded as embeddings.&quot;</span>
        <span class="p">)</span>
        <span class="k">if</span> <span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span>
            <span class="ow">and</span> <span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
            <span class="ow">and</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">)</span>
        <span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="n">transformer_error_message</span><span class="p">)</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols=</span><span class="si">{cat_embed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;continuous_cols=</span><span class="si">{continuous_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;quantization_setup=</span><span class="si">{quantization_setup}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cols_to_scale=</span><span class="si">{cols_to_scale}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;auto_embed_dim=</span><span class="si">{auto_embed_dim}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">!=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;embedding_rule=&#39;</span><span class="si">{embedding_rule}</span><span class="s2">&#39;&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">!=</span> <span class="mi">16</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;default_embed_dim=</span><span class="si">{default_embed_dim}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_attention=</span><span class="si">{with_attention}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_cls_token=</span><span class="si">{with_cls_token}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;shared_embed=</span><span class="si">{shared_embed}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;scale=</span><span class="si">{scale}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;already_standard=</span><span class="si">{already_standard}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
            <span class="p">)</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
            <span class="p">)</span>
        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;TabPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">









<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit</span>


<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Fits the Preprocessor and creates required attributes</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><code>TabPreprocessor</code> fitted object</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">312</span>
<span class="normal">313</span>
<span class="normal">314</span>
<span class="normal">315</span>
<span class="normal">316</span>
<span class="normal">317</span>
<span class="normal">318</span>
<span class="normal">319</span>
<span class="normal">320</span>
<span class="normal">321</span>
<span class="normal">322</span>
<span class="normal">323</span>
<span class="normal">324</span>
<span class="normal">325</span>
<span class="normal">326</span>
<span class="normal">327</span>
<span class="normal">328</span>
<span class="normal">329</span>
<span class="normal">330</span>
<span class="normal">331</span>
<span class="normal">332</span>
<span class="normal">333</span>
<span class="normal">334</span>
<span class="normal">335</span>
<span class="normal">336</span>
<span class="normal">337</span>
<span class="normal">338</span>
<span class="normal">339</span>
<span class="normal">340</span>
<span class="normal">341</span>
<span class="normal">342</span>
<span class="normal">343</span>
<span class="normal">344</span>
<span class="normal">345</span>
<span class="normal">346</span>
<span class="normal">347</span>
<span class="normal">348</span>
<span class="normal">349</span>
<span class="normal">350</span>
<span class="normal">351</span>
<span class="normal">352</span>
<span class="normal">353</span>
<span class="normal">354</span>
<span class="normal">355</span>
<span class="normal">356</span>
<span class="normal">357</span>
<span class="normal">358</span>
<span class="normal">359</span>
<span class="normal">360</span>
<span class="normal">361</span>
<span class="normal">362</span>
<span class="normal">363</span>
<span class="normal">364</span>
<span class="normal">365</span>
<span class="normal">366</span>
<span class="normal">367</span>
<span class="normal">368</span>
<span class="normal">369</span>
<span class="normal">370</span>
<span class="normal">371</span>
<span class="normal">372</span>
<span class="normal">373</span>
<span class="normal">374</span>
<span class="normal">375</span>
<span class="normal">376</span>
<span class="normal">377</span>
<span class="normal">378</span>
<span class="normal">379</span>
<span class="normal">380</span>
<span class="normal">381</span>
<span class="normal">382</span>
<span class="normal">383</span>
<span class="normal">384</span>
<span class="normal">385</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    TabPreprocessor</span>
<span class="sd">        `TabPreprocessor` fitted object</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

    <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>

    <span class="c1"># Categorical embeddings logic</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">(</span>
            <span class="p">[]</span>
        <span class="p">)</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">df_cat</span><span class="p">,</span> <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_categorical</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
            <span class="n">columns_to_encode</span><span class="o">=</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span>
            <span class="n">shared_embed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">,</span>
            <span class="n">with_attention</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">,</span>
        <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>

        <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)))</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="n">cat_embed_dim</span><span class="p">[</span><span class="n">k</span><span class="p">]))</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="p">)})</span>

    <span class="c1"># Continuous columns logic</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">df_cont</span><span class="p">,</span> <span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_continuous</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>

        <span class="c1"># Standardization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
            <span class="p">)</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Continuous columns will not be normalised&quot;</span><span class="p">)</span>

        <span class="c1"># Quantization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="c1"># we do not run &#39;Quantizer.fit&#39; here since in the wild case</span>
            <span class="c1"># someone wants standardization and quantization for the same</span>
            <span class="c1"># columns, the Quantizer will run on the scaled data</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>

            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">cont_embed_dim</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">))</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">cont_embed_dim</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
            <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)}</span>
        <span class="p">)</span>

    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

    <span class="k">return</span> <span class="bp">self</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">transform</span>


<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Returns the processed <code>dataframe</code> as a np.ndarray</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>transformed input dataframe</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">387</span>
<span class="normal">388</span>
<span class="normal">389</span>
<span class="normal">390</span>
<span class="normal">391</span>
<span class="normal">392</span>
<span class="normal">393</span>
<span class="normal">394</span>
<span class="normal">395</span>
<span class="normal">396</span>
<span class="normal">397</span>
<span class="normal">398</span>
<span class="normal">399</span>
<span class="normal">400</span>
<span class="normal">401</span>
<span class="normal">402</span>
<span class="normal">403</span>
<span class="normal">404</span>
<span class="normal">405</span>
<span class="normal">406</span>
<span class="normal">407</span>
<span class="normal">408</span>
<span class="normal">409</span>
<span class="normal">410</span>
<span class="normal">411</span>
<span class="normal">412</span>
<span class="normal">413</span>
<span class="normal">414</span>
<span class="normal">415</span>
<span class="normal">416</span>
<span class="normal">417</span>
<span class="normal">418</span>
<span class="normal">419</span>
<span class="normal">420</span>
<span class="normal">421</span>
<span class="normal">422</span>
<span class="normal">423</span>
<span class="normal">424</span>
<span class="normal">425</span>
<span class="normal">426</span>
<span class="normal">427</span>
<span class="normal">428</span>
<span class="normal">429</span>
<span class="normal">430</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the processed `dataframe` as a np.ndarray</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        transformed input dataframe</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">)</span>

    <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">df_cat</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">]</span>
        <span class="n">df_cat</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">df_cont</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>
        <span class="c1"># Standardization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">:</span>
            <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span>
                <span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span>
            <span class="p">)</span>
        <span class="c1"># Quantization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="c1"># Adjustment so I don&#39;t have to override the method</span>
            <span class="c1"># in &#39;ChunkTabPreprocessor&#39;</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
                <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">df_cont</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_cont</span><span class="p">)</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="n">df_deep</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">concat</span><span class="p">([</span><span class="n">df_cat</span><span class="p">,</span> <span class="n">df_cont</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
    <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cat</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="k">except</span> <span class="ne">NameError</span><span class="p">:</span>
            <span class="n">df_deep</span> <span class="o">=</span> <span class="n">df_cont</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

    <span class="k">return</span> <span class="n">df_deep</span><span class="o">.</span><span class="n">values</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">inverse_transform</span>


<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">encoded</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Takes as input the output from the <code>transform</code> method and it will
return the original values.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>encoded</code>
            </td>
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>array with the output of the <code>transform</code> method</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Pandas dataframe with the original values</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">435</span>
<span class="normal">436</span>
<span class="normal">437</span>
<span class="normal">438</span>
<span class="normal">439</span>
<span class="normal">440</span>
<span class="normal">441</span>
<span class="normal">442</span>
<span class="normal">443</span>
<span class="normal">444</span>
<span class="normal">445</span>
<span class="normal">446</span>
<span class="normal">447</span>
<span class="normal">448</span>
<span class="normal">449</span>
<span class="normal">450</span>
<span class="normal">451</span>
<span class="normal">452</span>
<span class="normal">453</span>
<span class="normal">454</span>
<span class="normal">455</span>
<span class="normal">456</span>
<span class="normal">457</span>
<span class="normal">458</span>
<span class="normal">459</span>
<span class="normal">460</span>
<span class="normal">461</span>
<span class="normal">462</span>
<span class="normal">463</span>
<span class="normal">464</span>
<span class="normal">465</span>
<span class="normal">466</span>
<span class="normal">467</span>
<span class="normal">468</span>
<span class="normal">469</span>
<span class="normal">470</span>
<span class="normal">471</span>
<span class="normal">472</span>
<span class="normal">473</span>
<span class="normal">474</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">encoded</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Takes as input the output from the `transform` method and it will</span>
<span class="sd">    return the original values.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    encoded: np.ndarray</span>
<span class="sd">        array with the output of the `transform` method</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    pd.DataFrame</span>
<span class="sd">        Pandas dataframe with the original values</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">decoded</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">encoded</span><span class="p">,</span> <span class="n">columns</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">keys</span><span class="p">()))</span>
    <span class="c1"># embeddings back to original category</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">decoded</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">decoded</span><span class="p">)</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="c1"># quantized cols to the mid point</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span>
                    <span class="s2">&quot;Note that quantized cols will be turned into the mid point of &quot;</span>
                    <span class="s2">&quot;the corresponding bin&quot;</span>
                <span class="p">)</span>
            <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span><span class="o">.</span><span class="n">inversed_bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span> <span class="o">=</span> <span class="n">decoded</span><span class="p">[</span><span class="n">k</span><span class="p">]</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">v</span><span class="p">)</span>
        <span class="c1"># continuous_cols back to non-standarised</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span>
                <span class="n">decoded</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span>
            <span class="p">)</span>
        <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>  <span class="c1"># KeyError:</span>
            <span class="k">pass</span>

    <span class="k">if</span> <span class="s2">&quot;cls_token&quot;</span> <span class="ow">in</span> <span class="n">decoded</span><span class="o">.</span><span class="n">columns</span><span class="p">:</span>
        <span class="n">decoded</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="s2">&quot;cls_token&quot;</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">decoded</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit_transform</span>


<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Combines <code>fit</code> and <code>transform</code></p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>transformed input dataframe</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">476</span>
<span class="normal">477</span>
<span class="normal">478</span>
<span class="normal">479</span>
<span class="normal">480</span>
<span class="normal">481</span>
<span class="normal">482</span>
<span class="normal">483</span>
<span class="normal">484</span>
<span class="normal">485</span>
<span class="normal">486</span>
<span class="normal">487</span>
<span class="normal">488</span>
<span class="normal">489</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        transformed input dataframe</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>



  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">Quantizer</span>


<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">


        <p>Helper class to perform the quantization of continuous columns. It is
included in this docs for completion, since depending on the value of the
parameter <code>'quantization_setup'</code> of the <code>TabPreprocessor</code> class, that
class might have an attribute of type <code>Quantizer</code>. However, this class is
designed to always run internally within the <code>TabPreprocessor</code> class.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>quantization_setup</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary where the keys are the column names to quantize and the
values are the either integers indicating the number of bins or a
list of scalars indicating the bin edges.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 39</span>
<span class="normal"> 40</span>
<span class="normal"> 41</span>
<span class="normal"> 42</span>
<span class="normal"> 43</span>
<span class="normal"> 44</span>
<span class="normal"> 45</span>
<span class="normal"> 46</span>
<span class="normal"> 47</span>
<span class="normal"> 48</span>
<span class="normal"> 49</span>
<span class="normal"> 50</span>
<span class="normal"> 51</span>
<span class="normal"> 52</span>
<span class="normal"> 53</span>
<span class="normal"> 54</span>
<span class="normal"> 55</span>
<span class="normal"> 56</span>
<span class="normal"> 57</span>
<span class="normal"> 58</span>
<span class="normal"> 59</span>
<span class="normal"> 60</span>
<span class="normal"> 61</span>
<span class="normal"> 62</span>
<span class="normal"> 63</span>
<span class="normal"> 64</span>
<span class="normal"> 65</span>
<span class="normal"> 66</span>
<span class="normal"> 67</span>
<span class="normal"> 68</span>
<span class="normal"> 69</span>
<span class="normal"> 70</span>
<span class="normal"> 71</span>
<span class="normal"> 72</span>
<span class="normal"> 73</span>
<span class="normal"> 74</span>
<span class="normal"> 75</span>
<span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">Quantizer</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Helper class to perform the quantization of continuous columns. It is</span>
<span class="sd">    included in this docs for completion, since depending on the value of the</span>
<span class="sd">    parameter `&#39;quantization_setup&#39;` of the `TabPreprocessor` class, that</span>
<span class="sd">    class might have an attribute of type `Quantizer`. However, this class is</span>
<span class="sd">    designed to always run internally within the `TabPreprocessor` class.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    quantization_setup: Dict, default = None</span>
<span class="sd">        Dictionary where the keys are the column names to quantize and the</span>
<span class="sd">        values are the either integers indicating the number of bins or a</span>
<span class="sd">        list of scalars indicating the bin edges.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]],</span>
        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span> <span class="o">=</span> <span class="n">kwargs</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;Quantizer&quot;</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
        <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">bins</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="n">_</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">cut</span><span class="p">(</span>
                <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">],</span> <span class="n">bins</span><span class="p">,</span> <span class="n">retbins</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span>
            <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">inversed_bins</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="nb">float</span><span class="p">]]</span> <span class="o">=</span> <span class="p">{}</span>
        <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">bins</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">inversed_bins</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="p">{</span>
                <span class="n">k</span><span class="p">:</span> <span class="n">v</span>
                <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span>
                    <span class="nb">zip</span><span class="p">(</span>
                        <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">bins</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">),</span>
                        <span class="p">[(</span><span class="n">a</span> <span class="o">+</span> <span class="n">b</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span> <span class="k">for</span> <span class="n">a</span><span class="p">,</span> <span class="n">b</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">bins</span><span class="p">,</span> <span class="n">bins</span><span class="p">[</span><span class="mi">1</span><span class="p">:])],</span>
                    <span class="p">)</span>
                <span class="p">)</span>
            <span class="p">}</span>
            <span class="c1"># 0</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">inversed_bins</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">nan</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">condition</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">)</span>

        <span class="n">dfc</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">bins</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">cut</span><span class="p">(</span><span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">],</span> <span class="n">bins</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>
            <span class="c1"># 0 will be left for numbers outside the bins, i.e. smaller than</span>
            <span class="c1"># the smaller boundary or larger than the largest boundary</span>
            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">+</span> <span class="mi">1</span>
            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">fillna</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
            <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">dfc</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>

        <span class="k">return</span> <span class="n">dfc</span>

    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;Quantizer(quantization_setup=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">











  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">TextPreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>


        <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>text_col</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>column in the input dataframe containing the texts</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>max_vocab</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Maximum number of tokens in the vocabulary</p>
              </div>
            </td>
            <td>
                  <code>30000</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>min_freq</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Minimum frequency for a token to be part of the vocabulary</p>
              </div>
            </td>
            <td>
                  <code>5</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>maxlen</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Maximum length of the tokenized sequences</p>
              </div>
            </td>
            <td>
                  <code>80</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>pad_first</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Indicates whether the padding index will be added at the beginning or the
end of the sequences</p>
              </div>
            </td>
            <td>
                  <code>True</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>pad_idx</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>already_processed</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[bool]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the sequence of elements is already processed or
prepared. If this is the case, this Preprocessor will simply tokenize
and pad the sequence. <br/></p>
<div class="highlight"><pre><span></span><code>Param aliases: `not_text`. &lt;br/&gt;
</code></pre></div>
<p>This parameter is thought for those cases where the input sequences
are already fully processed or are directly not text (e.g. IDs)</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>word_vectors_path</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Path to the pretrained word vectors</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>n_cpus</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>number of CPUs to used during the tokenization process</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>verbose</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Enable verbose output.</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.vocab">vocab</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.Vocab</code></p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.embedding_matrix">embedding_matrix</span></code></td>
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Array with the pretrained embeddings</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">TextPreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df_train</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;life is like a box of chocolates&quot;</span><span class="p">,</span>
<span class="gp">... </span><span class="s2">&quot;You never know what you&#39;re gonna get&quot;</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span> <span class="o">=</span> <span class="n">TextPreprocessor</span><span class="p">(</span><span class="n">text_col</span><span class="o">=</span><span class="s1">&#39;text_column&#39;</span><span class="p">,</span> <span class="n">max_vocab</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">min_freq</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">maxlen</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
<span class="go">The vocabulary contains 24 tokens</span>
<span class="go">array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],</span>
<span class="go">       [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df_te</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;you never know what is in the box&#39;</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">text_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_te</span><span class="p">)</span>
<span class="go">array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 22</span>
<span class="normal"> 23</span>
<span class="normal"> 24</span>
<span class="normal"> 25</span>
<span class="normal"> 26</span>
<span class="normal"> 27</span>
<span class="normal"> 28</span>
<span class="normal"> 29</span>
<span class="normal"> 30</span>
<span class="normal"> 31</span>
<span class="normal"> 32</span>
<span class="normal"> 33</span>
<span class="normal"> 34</span>
<span class="normal"> 35</span>
<span class="normal"> 36</span>
<span class="normal"> 37</span>
<span class="normal"> 38</span>
<span class="normal"> 39</span>
<span class="normal"> 40</span>
<span class="normal"> 41</span>
<span class="normal"> 42</span>
<span class="normal"> 43</span>
<span class="normal"> 44</span>
<span class="normal"> 45</span>
<span class="normal"> 46</span>
<span class="normal"> 47</span>
<span class="normal"> 48</span>
<span class="normal"> 49</span>
<span class="normal"> 50</span>
<span class="normal"> 51</span>
<span class="normal"> 52</span>
<span class="normal"> 53</span>
<span class="normal"> 54</span>
<span class="normal"> 55</span>
<span class="normal"> 56</span>
<span class="normal"> 57</span>
<span class="normal"> 58</span>
<span class="normal"> 59</span>
<span class="normal"> 60</span>
<span class="normal"> 61</span>
<span class="normal"> 62</span>
<span class="normal"> 63</span>
<span class="normal"> 64</span>
<span class="normal"> 65</span>
<span class="normal"> 66</span>
<span class="normal"> 67</span>
<span class="normal"> 68</span>
<span class="normal"> 69</span>
<span class="normal"> 70</span>
<span class="normal"> 71</span>
<span class="normal"> 72</span>
<span class="normal"> 73</span>
<span class="normal"> 74</span>
<span class="normal"> 75</span>
<span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span>
<span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span>
<span class="normal">207</span>
<span class="normal">208</span>
<span class="normal">209</span>
<span class="normal">210</span>
<span class="normal">211</span>
<span class="normal">212</span>
<span class="normal">213</span>
<span class="normal">214</span>
<span class="normal">215</span>
<span class="normal">216</span>
<span class="normal">217</span>
<span class="normal">218</span>
<span class="normal">219</span>
<span class="normal">220</span>
<span class="normal">221</span>
<span class="normal">222</span>
<span class="normal">223</span>
<span class="normal">224</span>
<span class="normal">225</span>
<span class="normal">226</span>
<span class="normal">227</span>
<span class="normal">228</span>
<span class="normal">229</span>
<span class="normal">230</span>
<span class="normal">231</span>
<span class="normal">232</span>
<span class="normal">233</span>
<span class="normal">234</span>
<span class="normal">235</span>
<span class="normal">236</span>
<span class="normal">237</span>
<span class="normal">238</span>
<span class="normal">239</span>
<span class="normal">240</span>
<span class="normal">241</span>
<span class="normal">242</span>
<span class="normal">243</span>
<span class="normal">244</span>
<span class="normal">245</span>
<span class="normal">246</span>
<span class="normal">247</span>
<span class="normal">248</span>
<span class="normal">249</span>
<span class="normal">250</span>
<span class="normal">251</span>
<span class="normal">252</span>
<span class="normal">253</span>
<span class="normal">254</span>
<span class="normal">255</span>
<span class="normal">256</span>
<span class="normal">257</span>
<span class="normal">258</span>
<span class="normal">259</span>
<span class="normal">260</span>
<span class="normal">261</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">TextPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the ``deeptext`` input dataset</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    text_col: str</span>
<span class="sd">        column in the input dataframe containing the texts</span>
<span class="sd">    max_vocab: int, default=30000</span>
<span class="sd">        Maximum number of tokens in the vocabulary</span>
<span class="sd">    min_freq: int, default=5</span>
<span class="sd">        Minimum frequency for a token to be part of the vocabulary</span>
<span class="sd">    maxlen: int, default=80</span>
<span class="sd">        Maximum length of the tokenized sequences</span>
<span class="sd">    pad_first: bool,  default = True</span>
<span class="sd">        Indicates whether the padding index will be added at the beginning or the</span>
<span class="sd">        end of the sequences</span>
<span class="sd">    pad_idx: int, default = 1</span>
<span class="sd">        padding index. Fastai&#39;s Tokenizer leaves 0 for the &#39;unknown&#39; token.</span>
<span class="sd">    already_processed: bool, Optional, default = False</span>
<span class="sd">        Boolean indicating if the sequence of elements is already processed or</span>
<span class="sd">        prepared. If this is the case, this Preprocessor will simply tokenize</span>
<span class="sd">        and pad the sequence. &lt;br/&gt;</span>

<span class="sd">            Param aliases: `not_text`. &lt;br/&gt;</span>

<span class="sd">        This parameter is thought for those cases where the input sequences</span>
<span class="sd">        are already fully processed or are directly not text (e.g. IDs)</span>
<span class="sd">    word_vectors_path: str, Optional</span>
<span class="sd">        Path to the pretrained word vectors</span>
<span class="sd">    n_cpus: int, Optional, default = None</span>
<span class="sd">        number of CPUs to used during the tokenization process</span>
<span class="sd">    verbose: int, default 1</span>
<span class="sd">        Enable verbose output.</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    vocab: Vocab</span>
<span class="sd">        an instance of `pytorch_widedeep.utils.fastai_transforms.Vocab`</span>
<span class="sd">    embedding_matrix: np.ndarray</span>
<span class="sd">        Array with the pretrained embeddings</span>

<span class="sd">    Examples</span>
<span class="sd">    ---------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import TextPreprocessor</span>
<span class="sd">    &gt;&gt;&gt; df_train = pd.DataFrame({&#39;text_column&#39;: [&quot;life is like a box of chocolates&quot;,</span>
<span class="sd">    ... &quot;You never know what you&#39;re gonna get&quot;]})</span>
<span class="sd">    &gt;&gt;&gt; text_preprocessor = TextPreprocessor(text_col=&#39;text_column&#39;, max_vocab=25, min_freq=1, maxlen=10)</span>
<span class="sd">    &gt;&gt;&gt; text_preprocessor.fit_transform(df_train)</span>
<span class="sd">    The vocabulary contains 24 tokens</span>
<span class="sd">    array([[ 1,  1,  1,  1, 10, 11, 12, 13, 14, 15],</span>
<span class="sd">           [ 5,  9, 16, 17, 18,  9, 19, 20, 21, 22]], dtype=int32)</span>
<span class="sd">    &gt;&gt;&gt; df_te = pd.DataFrame({&#39;text_column&#39;: [&#39;you never know what is in the box&#39;]})</span>
<span class="sd">    &gt;&gt;&gt; text_preprocessor.transform(df_te)</span>
<span class="sd">    array([[ 1,  1,  9, 16, 17, 18, 11,  0,  0, 13]], dtype=int32)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;already_processed&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;not_text&quot;</span><span class="p">])</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">30000</span><span class="p">,</span>
        <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
        <span class="n">maxlen</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">80</span><span class="p">,</span>
        <span class="n">pad_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
        <span class="n">pad_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
        <span class="n">already_processed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">word_vectors_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">TextPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="o">=</span> <span class="n">text_col</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span> <span class="o">=</span> <span class="n">max_vocab</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span> <span class="o">=</span> <span class="n">min_freq</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">maxlen</span> <span class="o">=</span> <span class="n">maxlen</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">pad_first</span> <span class="o">=</span> <span class="n">pad_first</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span> <span class="o">=</span> <span class="n">pad_idx</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span> <span class="o">=</span> <span class="n">already_processed</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="o">=</span> <span class="n">word_vectors_path</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="o">=</span> <span class="n">n_cpus</span> <span class="k">if</span> <span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Builds the vocabulary</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        TextPreprocessor</span>
<span class="sd">            `TextPreprocessor` fitted object</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">:</span> <span class="n">TVocab</span> <span class="o">=</span> <span class="n">Vocab</span><span class="p">(</span>
            <span class="n">max_vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">,</span>
            <span class="n">min_freq</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">,</span>
            <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
        <span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
            <span class="n">tokens</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The vocabulary contains </span><span class="si">{}</span><span class="s2"> tokens&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">)))</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">build_embeddings_matrix</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span>
            <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequences</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            Padded, _&#39;numericalised&#39;_ sequences</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequence</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        text: str</span>
<span class="sd">            text to be tokenized and padded</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            Padded, _&#39;numericalised&#39;_ sequence</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">([</span><span class="n">text</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            Padded, _&#39;numericalised&#39;_ sequences</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">padded_seq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Returns the original text plus the added &#39;special&#39; tokens</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        padded_seq: np.ndarray</span>
<span class="sd">            array with the output of the `transform` method</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        pd.DataFrame</span>
<span class="sd">            Pandas dataframe with the original text plus the added &#39;special&#39; tokens</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">padded_seq</span><span class="p">]</span>
        <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">:</span> <span class="n">texts</span><span class="p">})</span>

    <span class="k">def</span> <span class="nf">_pad_sequences</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tokens</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]])</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
        <span class="n">sequences</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">t</span><span class="p">)</span> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">tokens</span><span class="p">]</span>
        <span class="n">padded_seq</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
            <span class="p">[</span>
                <span class="n">pad_sequences</span><span class="p">(</span>
                    <span class="n">s</span><span class="p">,</span>
                    <span class="n">maxlen</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">maxlen</span><span class="p">,</span>
                    <span class="n">pad_first</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_first</span><span class="p">,</span>
                    <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
                <span class="p">)</span>
                <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">sequences</span>
            <span class="p">]</span>
        <span class="p">)</span>
        <span class="k">return</span> <span class="n">padded_seq</span>

    <span class="k">def</span> <span class="nf">_read_texts</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
        <span class="k">if</span> <span class="n">root_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">root_dir</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                    <span class="s2">&quot;root_dir does not exist. Please create it before fitting the preprocessor&quot;</span>
                <span class="p">)</span>
            <span class="n">texts_fnames</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
            <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">texts_fname</span> <span class="ow">in</span> <span class="n">texts_fnames</span><span class="p">:</span>
                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_dir</span><span class="p">,</span> <span class="n">texts_fname</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
                    <span class="n">texts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">texts</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>

        <span class="k">return</span> <span class="n">texts</span>

    <span class="k">def</span> <span class="nf">_load_vocab</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">vocab</span><span class="p">:</span> <span class="n">TVocab</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span> <span class="o">=</span> <span class="n">vocab</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;text_col=</span><span class="si">{text_col}</span><span class="s2">&quot;</span><span class="p">]</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;max_vocab=</span><span class="si">{max_vocab}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;min_freq=</span><span class="si">{min_freq}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;maxlen=</span><span class="si">{maxlen}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_first=</span><span class="si">{pad_first}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_idx=</span><span class="si">{pad_idx}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;already_processed=</span><span class="si">{already_processed}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;word_vectors_path=</span><span class="si">{word_vectors_path}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_cpus=</span><span class="si">{n_cpus}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;TextPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">









<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit</span>


<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Builds the vocabulary</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><code>TextPreprocessor</code> fitted object</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Builds the vocabulary</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    TextPreprocessor</span>
<span class="sd">        `TextPreprocessor` fitted object</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>

    <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">:</span> <span class="n">TVocab</span> <span class="o">=</span> <span class="n">Vocab</span><span class="p">(</span>
        <span class="n">max_vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">,</span>
        <span class="n">min_freq</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">,</span>
        <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
    <span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span>
        <span class="n">tokens</span><span class="p">,</span>
    <span class="p">)</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The vocabulary contains </span><span class="si">{}</span><span class="s2"> tokens&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">)))</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">build_embeddings_matrix</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span>
        <span class="p">)</span>

    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

    <span class="k">return</span> <span class="bp">self</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">transform</span>


<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Returns the padded, <em>'numericalised'</em> sequences</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Padded, <em>'numericalised'</em> sequences</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequences</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        Padded, _&#39;numericalised&#39;_ sequences</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">transform_sample</span>


<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Returns the padded, <em>'numericalised'</em> sequence</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>text</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>text to be tokenized and padded</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Padded, <em>'numericalised'</em> sequence</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the padded, _&#39;numericalised&#39;_ sequence</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    text: str</span>
<span class="sd">        text to be tokenized and padded</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        Padded, _&#39;numericalised&#39;_ sequence</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;vocab&quot;</span><span class="p">])</span>
    <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">([</span><span class="n">text</span><span class="p">],</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pad_sequences</span><span class="p">(</span><span class="n">tokens</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit_transform</span>


<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Combines <code>fit</code> and <code>transform</code></p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Padded, <em>'numericalised'</em> sequences</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        Padded, _&#39;numericalised&#39;_ sequences</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">inverse_transform</span>


<a href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">padded_seq</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Returns the original text plus the added 'special' tokens</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>padded_seq</code>
            </td>
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>array with the output of the <code>transform</code> method</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Pandas dataframe with the original text plus the added 'special' tokens</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span>
<span class="normal">207</span>
<span class="normal">208</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">padded_seq</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Returns the original text plus the added &#39;special&#39; tokens</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    padded_seq: np.ndarray</span>
<span class="sd">        array with the output of the `transform` method</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    pd.DataFrame</span>
<span class="sd">        Pandas dataframe with the original text plus the added &#39;special&#39; tokens</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">inverse_transform</span><span class="p">(</span><span class="n">num</span><span class="p">)</span> <span class="k">for</span> <span class="n">num</span> <span class="ow">in</span> <span class="n">padded_seq</span><span class="p">]</span>
    <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">:</span> <span class="n">texts</span><span class="p">})</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>



  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">HFPreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>


        <p>Text processor to prepare the <code>deeptext</code> input dataset that is a
wrapper around HuggingFace's tokenizers.</p>
<p>Following the main phylosophy of the <code>pytorch-widedeep</code> library, this
class is designed to be as flexible as possible. Therefore, it is coded
so that the user can use it as one would use any HuggingFace tokenizers,
or following the API call 'protocol' of the rest of the library.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>model_name</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
Currently supported models are those from the families: BERT, RoBERTa,
DistilBERT, ALBERT and ELECTRA.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>use_fast_tokenizer</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Whether to use the fast tokenizer from HuggingFace or not</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>text_col</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The column in the input dataframe containing the text data. If this
tokenizer is used via the <code>fit</code> and <code>transform</code> methods, this
argument is mandatory. If the tokenizer is used via the <code>encode</code>
method, this argument is not needed since the input text is passed
directly to the <code>encode</code> method.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>num_workers</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Number of workers to use when preprocessing the text data. If not
None, and <code>use_fast_tokenizer</code> is False, the text data will be
preprocessed in parallel using the number of workers specified. If
<code>use_fast_tokenizer</code> is True, this argument is ignored.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>preprocessing_rules</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>A list of functions to be applied to the text data before encoding.
This can be useful to clean the text data before encoding. For
example, removing html tags, special characters, etc.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>tokenizer_params</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Additional parameters to be passed to the HuggingFace's
<code>PreTrainedTokenizer</code>. Parameters to the <code>PreTrainedTokenizer</code>
can also be passed via the <code>**kwargs</code> argument</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>encode_params</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
of the HuggingFace's <code>PreTrainedTokenizer</code>. If the <code>fit</code> and <code>transform</code>
methods are used, the <code>encode_params</code> dict parameter is mandatory. If
the <code>encode</code> method is used, this parameter is not needed since the
input text is passed directly to the <code>encode</code> method.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>**kwargs</code>
            </td>
            <td>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Additional kwargs to be passed to the model, in particular to the
<code>PreTrainedTokenizer</code> class.</p>
              </div>
            </td>
            <td>
                  <code>{}</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.is_fitted">is_fitted</span></code></td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the preprocessor has been fitted. This is a
HuggingFacea tokenizer, so it is always considered fitted and this
attribute is manually set to True internally. This parameter exists
for consistency with the rest of the library and because is needed
for some functionality in the library.</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">HFPreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s2">&quot;text&quot;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;this is the first text&quot;</span><span class="p">,</span> <span class="s2">&quot;this is the second text&quot;</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hf_processor_1</span> <span class="o">=</span> <span class="n">HFPreprocessor</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;bert-base-uncased&quot;</span><span class="p">,</span> <span class="n">text_col</span><span class="o">=</span><span class="s2">&quot;text&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_1</span> <span class="o">=</span> <span class="n">hf_processor_1</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;this is a new text&quot;</span><span class="p">,</span> <span class="s2">&quot;this is another text&quot;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hf_processor_2</span> <span class="o">=</span> <span class="n">HFPreprocessor</span><span class="p">(</span><span class="n">model_name</span><span class="o">=</span><span class="s2">&quot;bert-base-uncased&quot;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_text_2</span> <span class="o">=</span> <span class="n">hf_processor_2</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="n">max_length</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">padding</span><span class="o">=</span><span class="s2">&quot;max_length&quot;</span><span class="p">,</span> <span class="n">truncation</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 17</span>
<span class="normal"> 18</span>
<span class="normal"> 19</span>
<span class="normal"> 20</span>
<span class="normal"> 21</span>
<span class="normal"> 22</span>
<span class="normal"> 23</span>
<span class="normal"> 24</span>
<span class="normal"> 25</span>
<span class="normal"> 26</span>
<span class="normal"> 27</span>
<span class="normal"> 28</span>
<span class="normal"> 29</span>
<span class="normal"> 30</span>
<span class="normal"> 31</span>
<span class="normal"> 32</span>
<span class="normal"> 33</span>
<span class="normal"> 34</span>
<span class="normal"> 35</span>
<span class="normal"> 36</span>
<span class="normal"> 37</span>
<span class="normal"> 38</span>
<span class="normal"> 39</span>
<span class="normal"> 40</span>
<span class="normal"> 41</span>
<span class="normal"> 42</span>
<span class="normal"> 43</span>
<span class="normal"> 44</span>
<span class="normal"> 45</span>
<span class="normal"> 46</span>
<span class="normal"> 47</span>
<span class="normal"> 48</span>
<span class="normal"> 49</span>
<span class="normal"> 50</span>
<span class="normal"> 51</span>
<span class="normal"> 52</span>
<span class="normal"> 53</span>
<span class="normal"> 54</span>
<span class="normal"> 55</span>
<span class="normal"> 56</span>
<span class="normal"> 57</span>
<span class="normal"> 58</span>
<span class="normal"> 59</span>
<span class="normal"> 60</span>
<span class="normal"> 61</span>
<span class="normal"> 62</span>
<span class="normal"> 63</span>
<span class="normal"> 64</span>
<span class="normal"> 65</span>
<span class="normal"> 66</span>
<span class="normal"> 67</span>
<span class="normal"> 68</span>
<span class="normal"> 69</span>
<span class="normal"> 70</span>
<span class="normal"> 71</span>
<span class="normal"> 72</span>
<span class="normal"> 73</span>
<span class="normal"> 74</span>
<span class="normal"> 75</span>
<span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span>
<span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span>
<span class="normal">207</span>
<span class="normal">208</span>
<span class="normal">209</span>
<span class="normal">210</span>
<span class="normal">211</span>
<span class="normal">212</span>
<span class="normal">213</span>
<span class="normal">214</span>
<span class="normal">215</span>
<span class="normal">216</span>
<span class="normal">217</span>
<span class="normal">218</span>
<span class="normal">219</span>
<span class="normal">220</span>
<span class="normal">221</span>
<span class="normal">222</span>
<span class="normal">223</span>
<span class="normal">224</span>
<span class="normal">225</span>
<span class="normal">226</span>
<span class="normal">227</span>
<span class="normal">228</span>
<span class="normal">229</span>
<span class="normal">230</span>
<span class="normal">231</span>
<span class="normal">232</span>
<span class="normal">233</span>
<span class="normal">234</span>
<span class="normal">235</span>
<span class="normal">236</span>
<span class="normal">237</span>
<span class="normal">238</span>
<span class="normal">239</span>
<span class="normal">240</span>
<span class="normal">241</span>
<span class="normal">242</span>
<span class="normal">243</span>
<span class="normal">244</span>
<span class="normal">245</span>
<span class="normal">246</span>
<span class="normal">247</span>
<span class="normal">248</span>
<span class="normal">249</span>
<span class="normal">250</span>
<span class="normal">251</span>
<span class="normal">252</span>
<span class="normal">253</span>
<span class="normal">254</span>
<span class="normal">255</span>
<span class="normal">256</span>
<span class="normal">257</span>
<span class="normal">258</span>
<span class="normal">259</span>
<span class="normal">260</span>
<span class="normal">261</span>
<span class="normal">262</span>
<span class="normal">263</span>
<span class="normal">264</span>
<span class="normal">265</span>
<span class="normal">266</span>
<span class="normal">267</span>
<span class="normal">268</span>
<span class="normal">269</span>
<span class="normal">270</span>
<span class="normal">271</span>
<span class="normal">272</span>
<span class="normal">273</span>
<span class="normal">274</span>
<span class="normal">275</span>
<span class="normal">276</span>
<span class="normal">277</span>
<span class="normal">278</span>
<span class="normal">279</span>
<span class="normal">280</span>
<span class="normal">281</span>
<span class="normal">282</span>
<span class="normal">283</span>
<span class="normal">284</span>
<span class="normal">285</span>
<span class="normal">286</span>
<span class="normal">287</span>
<span class="normal">288</span>
<span class="normal">289</span>
<span class="normal">290</span>
<span class="normal">291</span>
<span class="normal">292</span>
<span class="normal">293</span>
<span class="normal">294</span>
<span class="normal">295</span>
<span class="normal">296</span>
<span class="normal">297</span>
<span class="normal">298</span>
<span class="normal">299</span>
<span class="normal">300</span>
<span class="normal">301</span>
<span class="normal">302</span>
<span class="normal">303</span>
<span class="normal">304</span>
<span class="normal">305</span>
<span class="normal">306</span>
<span class="normal">307</span>
<span class="normal">308</span>
<span class="normal">309</span>
<span class="normal">310</span>
<span class="normal">311</span>
<span class="normal">312</span>
<span class="normal">313</span>
<span class="normal">314</span>
<span class="normal">315</span>
<span class="normal">316</span>
<span class="normal">317</span>
<span class="normal">318</span>
<span class="normal">319</span>
<span class="normal">320</span>
<span class="normal">321</span>
<span class="normal">322</span>
<span class="normal">323</span>
<span class="normal">324</span>
<span class="normal">325</span>
<span class="normal">326</span>
<span class="normal">327</span>
<span class="normal">328</span>
<span class="normal">329</span>
<span class="normal">330</span>
<span class="normal">331</span>
<span class="normal">332</span>
<span class="normal">333</span>
<span class="normal">334</span>
<span class="normal">335</span>
<span class="normal">336</span>
<span class="normal">337</span>
<span class="normal">338</span>
<span class="normal">339</span>
<span class="normal">340</span>
<span class="normal">341</span>
<span class="normal">342</span>
<span class="normal">343</span>
<span class="normal">344</span>
<span class="normal">345</span>
<span class="normal">346</span>
<span class="normal">347</span>
<span class="normal">348</span>
<span class="normal">349</span>
<span class="normal">350</span>
<span class="normal">351</span>
<span class="normal">352</span>
<span class="normal">353</span>
<span class="normal">354</span>
<span class="normal">355</span>
<span class="normal">356</span>
<span class="normal">357</span>
<span class="normal">358</span>
<span class="normal">359</span>
<span class="normal">360</span>
<span class="normal">361</span>
<span class="normal">362</span>
<span class="normal">363</span>
<span class="normal">364</span>
<span class="normal">365</span>
<span class="normal">366</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">HFPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Text processor to prepare the ``deeptext`` input dataset that is a</span>
<span class="sd">    wrapper around HuggingFace&#39;s tokenizers.</span>

<span class="sd">    Following the main phylosophy of the `pytorch-widedeep` library, this</span>
<span class="sd">    class is designed to be as flexible as possible. Therefore, it is coded</span>
<span class="sd">    so that the user can use it as one would use any HuggingFace tokenizers,</span>
<span class="sd">    or following the API call &#39;protocol&#39; of the rest of the library.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    model_name: str</span>
<span class="sd">        The model name from the transformers library e.g. _&#39;bert-base-uncased&#39;_.</span>
<span class="sd">        Currently supported models are those from the families: BERT, RoBERTa,</span>
<span class="sd">        DistilBERT, ALBERT and ELECTRA.</span>
<span class="sd">    use_fast_tokenizer: bool, default = False</span>
<span class="sd">        Whether to use the fast tokenizer from HuggingFace or not</span>
<span class="sd">    text_col: Optional[str], default = None</span>
<span class="sd">        The column in the input dataframe containing the text data. If this</span>
<span class="sd">        tokenizer is used via the `fit` and `transform` methods, this</span>
<span class="sd">        argument is mandatory. If the tokenizer is used via the `encode`</span>
<span class="sd">        method, this argument is not needed since the input text is passed</span>
<span class="sd">        directly to the `encode` method.</span>
<span class="sd">    num_workers: Optional[int], default = None</span>
<span class="sd">        Number of workers to use when preprocessing the text data. If not</span>
<span class="sd">        None, and `use_fast_tokenizer` is False, the text data will be</span>
<span class="sd">        preprocessed in parallel using the number of workers specified. If</span>
<span class="sd">        `use_fast_tokenizer` is True, this argument is ignored.</span>
<span class="sd">    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None</span>
<span class="sd">        A list of functions to be applied to the text data before encoding.</span>
<span class="sd">        This can be useful to clean the text data before encoding. For</span>
<span class="sd">        example, removing html tags, special characters, etc.</span>
<span class="sd">    tokenizer_params: Optional[Dict[str, Any]], default = None</span>
<span class="sd">        Additional parameters to be passed to the HuggingFace&#39;s</span>
<span class="sd">        `PreTrainedTokenizer`. Parameters to the `PreTrainedTokenizer`</span>
<span class="sd">        can also be passed via the `**kwargs` argument</span>
<span class="sd">    encode_params: Optional[Dict[str, Any]], default = None</span>
<span class="sd">        Additional parameters to be passed to the `batch_encode_plus` method</span>
<span class="sd">        of the HuggingFace&#39;s `PreTrainedTokenizer`. If the `fit` and `transform`</span>
<span class="sd">        methods are used, the `encode_params` dict parameter is mandatory. If</span>
<span class="sd">        the `encode` method is used, this parameter is not needed since the</span>
<span class="sd">        input text is passed directly to the `encode` method.</span>
<span class="sd">    **kwargs</span>
<span class="sd">        Additional kwargs to be passed to the model, in particular to the</span>
<span class="sd">        `PreTrainedTokenizer` class.</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    is_fitted: bool</span>
<span class="sd">        Boolean indicating if the preprocessor has been fitted. This is a</span>
<span class="sd">        HuggingFacea tokenizer, so it is always considered fitted and this</span>
<span class="sd">        attribute is manually set to True internally. This parameter exists</span>
<span class="sd">        for consistency with the rest of the library and because is needed</span>
<span class="sd">        for some functionality in the library.</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import HFPreprocessor</span>
<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame({&quot;text&quot;: [&quot;this is the first text&quot;, &quot;this is the second text&quot;]})</span>
<span class="sd">    &gt;&gt;&gt; hf_processor_1 = HFPreprocessor(model_name=&quot;bert-base-uncased&quot;, text_col=&quot;text&quot;)</span>
<span class="sd">    &gt;&gt;&gt; X_text_1 = hf_processor_1.fit_transform(df)</span>
<span class="sd">    &gt;&gt;&gt; texts = [&quot;this is a new text&quot;, &quot;this is another text&quot;]</span>
<span class="sd">    &gt;&gt;&gt; hf_processor_2 = HFPreprocessor(model_name=&quot;bert-base-uncased&quot;)</span>
<span class="sd">    &gt;&gt;&gt; X_text_2 = hf_processor_2.encode(texts, max_length=10, padding=&quot;max_length&quot;, truncation=True)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="o">*</span><span class="p">,</span>
        <span class="n">use_fast_tokenizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">text_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">num_workers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">preprocessing_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">tokenizer_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">encode_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">model_name</span> <span class="o">=</span> <span class="n">model_name</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span> <span class="o">=</span> <span class="n">use_fast_tokenizer</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="o">=</span> <span class="n">text_col</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="o">=</span> <span class="n">num_workers</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span> <span class="o">=</span> <span class="n">preprocessing_rules</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span> <span class="o">=</span> <span class="n">tokenizer_params</span> <span class="k">if</span> <span class="n">tokenizer_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">{}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="o">=</span> <span class="n">encode_params</span> <span class="k">if</span> <span class="n">encode_params</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="p">{}</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span> <span class="o">=</span> <span class="p">(</span>
            <span class="n">num_workers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">num_workers</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">use_fast_tokenizer</span>
        <span class="p">)</span>

        <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span> <span class="o">=</span> <span class="n">get_tokenizer</span><span class="p">(</span>
            <span class="n">model_name</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="p">,</span>
            <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span><span class="p">,</span>
            <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="c1"># A HuggingFace tokenizer is already trained, since we need this</span>
        <span class="c1"># attribute elsewhere in the library, we simply set it to True</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

    <span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Encodes a list of texts. The method is a wrapper around the</span>
<span class="sd">        `batch_encode_plus` method of the HuggingFace&#39;s tokenizer.</span>

<span class="sd">        if &#39;use_fast_tokenizer&#39; is True, the method will use the `batch_encode_plus`</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        texts: List[str]</span>
<span class="sd">            List of texts to be encoded</span>
<span class="sd">        **kwargs</span>
<span class="sd">            Additional parameters to be passed to the `batch_encode_plus` method</span>
<span class="sd">            of the HuggingFace&#39;s tokenizer. If the &#39;encode_params&#39; dict was passed</span>
<span class="sd">            when instantiating the class, that dictionaly will be updated with</span>
<span class="sd">            the kwargs passed here.</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.array</span>
<span class="sd">            The encoded texts</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
                <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_text_parallel</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_preprocess_text</span><span class="p">(</span><span class="n">text</span><span class="p">)</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="n">texts</span><span class="p">]</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
            <span class="n">input_ids</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_encode_paralell</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">encoded_texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">batch_encode_plus</span><span class="p">(</span>
                <span class="n">texts</span><span class="p">,</span>
                <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">,</span>
            <span class="p">)</span>
            <span class="n">input_ids</span> <span class="o">=</span> <span class="n">encoded_texts</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;input_ids&quot;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">try</span><span class="p">:</span>
            <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">input_ids</span><span class="p">)</span>
        <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
                <span class="s2">&quot;Padding and Truncating parameters were not passed and all input arrays &quot;</span>
                <span class="s2">&quot;do not have the same shape. Padding to the longest sequence. &quot;</span>
                <span class="s2">&quot;Padding will be done with the index of the pad token for the model&quot;</span><span class="p">,</span>
                <span class="ne">UserWarning</span><span class="p">,</span>
            <span class="p">)</span>
            <span class="n">max_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)</span> <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span><span class="p">])</span>
            <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
                <span class="p">[</span>
                    <span class="n">np</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">ids</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">pad_token_id</span><span class="p">,</span> <span class="n">max_len</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)))</span>
                    <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span>
                <span class="p">]</span>
            <span class="p">)</span>

        <span class="k">return</span> <span class="n">output</span>

    <span class="k">def</span> <span class="nf">decode</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Decodes a list of input_ids. The method is a wrapper around the</span>
<span class="sd">        `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the</span>
<span class="sd">        HuggingFace&#39;s tokenizer.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        input_ids: npt.NDArray[np.int64]</span>
<span class="sd">            The input_ids to be decoded</span>
<span class="sd">        skip_special_tokens: bool</span>
<span class="sd">            Whether to skip the special tokens or not</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        List[str]</span>
<span class="sd">            The decoded texts</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_tokens_to_string</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_ids_to_tokens</span><span class="p">(</span><span class="n">input_ids</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
            <span class="p">)</span>
            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">input_ids</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
        <span class="p">]</span>
        <span class="k">return</span> <span class="n">texts</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;HFPreprocessor&quot;</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        This method is included for consistency with the rest of the library</span>
<span class="sd">        in general and with the `BasePreprocessor` in particular. HuggingFace&#39;s</span>
<span class="sd">        tokenizers and models are already trained. Therefore, the &#39;fit&#39; method</span>
<span class="sd">        here does nothing other than checking that the &#39;text_col&#39; parameter is</span>
<span class="sd">        not `None`.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            The dataframe containing the text data in the column specified by</span>
<span class="sd">            the &#39;text_col&#39; parameter</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
                <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
            <span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Encodes the text data in the input dataframe. This method simply</span>
<span class="sd">        calls the `encode` method under the hood. Similar to the `fit` method,</span>
<span class="sd">        this method is included for consistency with the rest of the library</span>
<span class="sd">        in general and with the `BasePreprocessor` in particular.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            The dataframe containing the text data in the column specified by</span>
<span class="sd">            the &#39;text_col&#39; parameter</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.array</span>
<span class="sd">            The encoded texts</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
                <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
            <span class="p">)</span>

        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="p">)</span>

        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Encodes a single text sample.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        text: str</span>
<span class="sd">            The text sample to be encoded</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.array</span>
<span class="sd">            The encoded text</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;The `encode` (or `fit`) method must be called before calling `transform_sample`&quot;</span>
            <span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">([</span><span class="n">text</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Encodes the text data in the input dataframe.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            The dataframe containing the text data in the column specified by</span>
<span class="sd">            the &#39;text_col&#39; parameter</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.array</span>
<span class="sd">            The encoded texts</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Decodes a list of input_ids. The method simply calls the `decode` method</span>
<span class="sd">        under the hood.</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        input_ids: npt.NDArray[np.int64]</span>
<span class="sd">            The input_ids to be decoded</span>
<span class="sd">        skip_special_tokens: bool</span>
<span class="sd">            Whether to skip the special tokens or not</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        List[str]</span>
<span class="sd">            The decoded texts</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_process_text_parallel</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
        <span class="n">num_processes</span> <span class="o">=</span> <span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
        <span class="p">)</span>
        <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">num_processes</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
            <span class="n">processed_texts</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_preprocess_text</span><span class="p">,</span> <span class="n">texts</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">processed_texts</span>

    <span class="k">def</span> <span class="nf">_preprocess_text</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="k">for</span> <span class="n">rule</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="p">:</span>
            <span class="n">text</span> <span class="o">=</span> <span class="n">rule</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">text</span>

    <span class="k">def</span> <span class="nf">_encode_paralell</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]:</span>
        <span class="n">num_processes</span> <span class="o">=</span> <span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">os</span><span class="o">.</span><span class="n">cpu_count</span><span class="p">()</span>
        <span class="p">)</span>
        <span class="n">ptokenizer</span> <span class="o">=</span> <span class="n">partial</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">encode</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
        <span class="k">with</span> <span class="n">ProcessPoolExecutor</span><span class="p">(</span><span class="n">max_workers</span><span class="o">=</span><span class="n">num_processes</span><span class="p">)</span> <span class="k">as</span> <span class="n">executor</span><span class="p">:</span>
            <span class="n">encoded_texts</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">executor</span><span class="o">.</span><span class="n">map</span><span class="p">(</span><span class="n">ptokenizer</span><span class="p">,</span> <span class="n">texts</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">encoded_texts</span>

    <span class="k">def</span> <span class="nf">_read_texts</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
        <span class="k">if</span> <span class="n">root_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">exists</span><span class="p">(</span><span class="n">root_dir</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                    <span class="s2">&quot;root_dir does not exist. Please create it before fitting the preprocessor&quot;</span>
                <span class="p">)</span>
            <span class="n">texts_fnames</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
            <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">texts_fname</span> <span class="ow">in</span> <span class="n">texts_fnames</span><span class="p">:</span>
                <span class="k">with</span> <span class="nb">open</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">root_dir</span><span class="p">,</span> <span class="n">texts_fname</span><span class="p">),</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
                    <span class="n">texts</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">read</span><span class="p">()</span><span class="o">.</span><span class="n">replace</span><span class="p">(</span><span class="s2">&quot;</span><span class="se">\n</span><span class="s2">&quot;</span><span class="p">,</span> <span class="s2">&quot;&quot;</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">texts</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>

        <span class="k">return</span> <span class="n">texts</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">(</span>
            <span class="sa">f</span><span class="s2">&quot;HFPreprocessor(text_col=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="si">}</span><span class="s2">, model_name=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="si">}</span><span class="s2">, &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;use_fast_tokenizer=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span><span class="si">}</span><span class="s2">, num_workers=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="si">}</span><span class="s2">, &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;preprocessing_rules=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="si">}</span><span class="s2">, tokenizer_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="si">}</span><span class="s2">, &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;encode_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="si">}</span><span class="s2">)&quot;</span>
        <span class="p">)</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">









<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">encode</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.encode" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Encodes a list of texts. The method is a wrapper around the
<code>batch_encode_plus</code> method of the HuggingFace's tokenizer.</p>
<p>if 'use_fast_tokenizer' is True, the method will use the <code>batch_encode_plus</code></p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>texts</code>
            </td>
            <td>
                  <code><span title="typing.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of texts to be encoded</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>**kwargs</code>
            </td>
            <td>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
of the HuggingFace's tokenizer. If the 'encode_params' dict was passed
when instantiating the class, that dictionaly will be updated with
the kwargs passed here.</p>
              </div>
            </td>
            <td>
                  <code>{}</code>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.array">array</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The encoded texts</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">encode</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">texts</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Encodes a list of texts. The method is a wrapper around the</span>
<span class="sd">    `batch_encode_plus` method of the HuggingFace&#39;s tokenizer.</span>

<span class="sd">    if &#39;use_fast_tokenizer&#39; is True, the method will use the `batch_encode_plus`</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    texts: List[str]</span>
<span class="sd">        List of texts to be encoded</span>
<span class="sd">    **kwargs</span>
<span class="sd">        Additional parameters to be passed to the `batch_encode_plus` method</span>
<span class="sd">        of the HuggingFace&#39;s tokenizer. If the &#39;encode_params&#39; dict was passed</span>
<span class="sd">        when instantiating the class, that dictionaly will be updated with</span>
<span class="sd">        the kwargs passed here.</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.array</span>
<span class="sd">        The encoded texts</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">kwargs</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">kwargs</span><span class="p">)</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="p">:</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
            <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_process_text_parallel</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_preprocess_text</span><span class="p">(</span><span class="n">text</span><span class="p">)</span> <span class="k">for</span> <span class="n">text</span> <span class="ow">in</span> <span class="n">texts</span><span class="p">]</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_multiprocessing</span><span class="p">:</span>
        <span class="n">input_ids</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_encode_paralell</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">encoded_texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">batch_encode_plus</span><span class="p">(</span>
            <span class="n">texts</span><span class="p">,</span>
            <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">,</span>
        <span class="p">)</span>
        <span class="n">input_ids</span> <span class="o">=</span> <span class="n">encoded_texts</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;input_ids&quot;</span><span class="p">)</span>

    <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

    <span class="k">try</span><span class="p">:</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">input_ids</span><span class="p">)</span>
    <span class="k">except</span> <span class="ne">ValueError</span><span class="p">:</span>
        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
            <span class="s2">&quot;Padding and Truncating parameters were not passed and all input arrays &quot;</span>
            <span class="s2">&quot;do not have the same shape. Padding to the longest sequence. &quot;</span>
            <span class="s2">&quot;Padding will be done with the index of the pad token for the model&quot;</span><span class="p">,</span>
            <span class="ne">UserWarning</span><span class="p">,</span>
        <span class="p">)</span>
        <span class="n">max_len</span> <span class="o">=</span> <span class="nb">max</span><span class="p">([</span><span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)</span> <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span><span class="p">])</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span>
            <span class="p">[</span>
                <span class="n">np</span><span class="o">.</span><span class="n">pad</span><span class="p">(</span><span class="n">ids</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">pad_token_id</span><span class="p">,</span> <span class="n">max_len</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">ids</span><span class="p">)))</span>
                <span class="k">for</span> <span class="n">ids</span> <span class="ow">in</span> <span class="n">input_ids</span>
            <span class="p">]</span>
        <span class="p">)</span>

    <span class="k">return</span> <span class="n">output</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">decode</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.decode" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Decodes a list of input_ids. The method is a wrapper around the
<code>convert_ids_to_tokens</code> and <code>convert_tokens_to_string</code> methods of the
HuggingFace's tokenizer.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>input_ids</code>
            </td>
            <td>
                  <code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The input_ids to be decoded</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>skip_special_tokens</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Whether to skip the special tokens or not</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="typing.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The decoded texts</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span>
<span class="normal">207</span>
<span class="normal">208</span>
<span class="normal">209</span>
<span class="normal">210</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">decode</span><span class="p">(</span>
    <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Decodes a list of input_ids. The method is a wrapper around the</span>
<span class="sd">    `convert_ids_to_tokens` and `convert_tokens_to_string` methods of the</span>
<span class="sd">    HuggingFace&#39;s tokenizer.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    input_ids: npt.NDArray[np.int64]</span>
<span class="sd">        The input_ids to be decoded</span>
<span class="sd">    skip_special_tokens: bool</span>
<span class="sd">        Whether to skip the special tokens or not</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    List[str]</span>
<span class="sd">        The decoded texts</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">texts</span> <span class="o">=</span> <span class="p">[</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_tokens_to_string</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">tokenizer</span><span class="o">.</span><span class="n">convert_ids_to_tokens</span><span class="p">(</span><span class="n">input_ids</span><span class="p">[</span><span class="n">i</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
        <span class="p">)</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">input_ids</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
    <span class="p">]</span>
    <span class="k">return</span> <span class="n">texts</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>This method is included for consistency with the rest of the library
in general and with the <code>BasePreprocessor</code> in particular. HuggingFace's
tokenizers and models are already trained. Therefore, the 'fit' method
here does nothing other than checking that the 'text_col' parameter is
not <code>None</code>.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The dataframe containing the text data in the column specified by
the 'text_col' parameter</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">212</span>
<span class="normal">213</span>
<span class="normal">214</span>
<span class="normal">215</span>
<span class="normal">216</span>
<span class="normal">217</span>
<span class="normal">218</span>
<span class="normal">219</span>
<span class="normal">220</span>
<span class="normal">221</span>
<span class="normal">222</span>
<span class="normal">223</span>
<span class="normal">224</span>
<span class="normal">225</span>
<span class="normal">226</span>
<span class="normal">227</span>
<span class="normal">228</span>
<span class="normal">229</span>
<span class="normal">230</span>
<span class="normal">231</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;HFPreprocessor&quot;</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    This method is included for consistency with the rest of the library</span>
<span class="sd">    in general and with the `BasePreprocessor` in particular. HuggingFace&#39;s</span>
<span class="sd">    tokenizers and models are already trained. Therefore, the &#39;fit&#39; method</span>
<span class="sd">    here does nothing other than checking that the &#39;text_col&#39; parameter is</span>
<span class="sd">    not `None`.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        The dataframe containing the text data in the column specified by</span>
<span class="sd">        the &#39;text_col&#39; parameter</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
            <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
            <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
        <span class="p">)</span>
    <span class="k">return</span> <span class="bp">self</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">transform</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Encodes the text data in the input dataframe. This method simply
calls the <code>encode</code> method under the hood. Similar to the <code>fit</code> method,
this method is included for consistency with the rest of the library
in general and with the <code>BasePreprocessor</code> in particular.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The dataframe containing the text data in the column specified by
the 'text_col' parameter</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.array">array</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The encoded texts</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">233</span>
<span class="normal">234</span>
<span class="normal">235</span>
<span class="normal">236</span>
<span class="normal">237</span>
<span class="normal">238</span>
<span class="normal">239</span>
<span class="normal">240</span>
<span class="normal">241</span>
<span class="normal">242</span>
<span class="normal">243</span>
<span class="normal">244</span>
<span class="normal">245</span>
<span class="normal">246</span>
<span class="normal">247</span>
<span class="normal">248</span>
<span class="normal">249</span>
<span class="normal">250</span>
<span class="normal">251</span>
<span class="normal">252</span>
<span class="normal">253</span>
<span class="normal">254</span>
<span class="normal">255</span>
<span class="normal">256</span>
<span class="normal">257</span>
<span class="normal">258</span>
<span class="normal">259</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Encodes the text data in the input dataframe. This method simply</span>
<span class="sd">    calls the `encode` method under the hood. Similar to the `fit` method,</span>
<span class="sd">    this method is included for consistency with the rest of the library</span>
<span class="sd">    in general and with the `BasePreprocessor` in particular.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        The dataframe containing the text data in the column specified by</span>
<span class="sd">        the &#39;text_col&#39; parameter</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.array</span>
<span class="sd">        The encoded texts</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">text_col</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
            <span class="s2">&quot;&#39;text_col&#39; is None. Please specify the column name containing the text data&quot;</span>
            <span class="s2">&quot; if you want to use the &#39;fit&#39; method&quot;</span>
        <span class="p">)</span>

    <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="p">)</span>

    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">texts</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">transform_sample</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.transform_sample" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform_sample</span><span class="p">(</span><span class="n">text</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Encodes a single text sample.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>text</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The text sample to be encoded</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.array">array</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The encoded text</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">261</span>
<span class="normal">262</span>
<span class="normal">263</span>
<span class="normal">264</span>
<span class="normal">265</span>
<span class="normal">266</span>
<span class="normal">267</span>
<span class="normal">268</span>
<span class="normal">269</span>
<span class="normal">270</span>
<span class="normal">271</span>
<span class="normal">272</span>
<span class="normal">273</span>
<span class="normal">274</span>
<span class="normal">275</span>
<span class="normal">276</span>
<span class="normal">277</span>
<span class="normal">278</span>
<span class="normal">279</span>
<span class="normal">280</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">text</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Encodes a single text sample.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    text: str</span>
<span class="sd">        The text sample to be encoded</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.array</span>
<span class="sd">        The encoded text</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
            <span class="s2">&quot;The `encode` (or `fit`) method must be called before calling `transform_sample`&quot;</span>
        <span class="p">)</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode</span><span class="p">([</span><span class="n">text</span><span class="p">])[</span><span class="mi">0</span><span class="p">]</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit_transform</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Encodes the text data in the input dataframe.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The dataframe containing the text data in the column specified by
the 'text_col' parameter</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.array">array</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The encoded texts</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">282</span>
<span class="normal">283</span>
<span class="normal">284</span>
<span class="normal">285</span>
<span class="normal">286</span>
<span class="normal">287</span>
<span class="normal">288</span>
<span class="normal">289</span>
<span class="normal">290</span>
<span class="normal">291</span>
<span class="normal">292</span>
<span class="normal">293</span>
<span class="normal">294</span>
<span class="normal">295</span>
<span class="normal">296</span>
<span class="normal">297</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Encodes the text data in the input dataframe.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        The dataframe containing the text data in the column specified by</span>
<span class="sd">        the &#39;text_col&#39; parameter</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.array</span>
<span class="sd">        The encoded texts</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">inverse_transform</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor.inverse_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">inverse_transform</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Decodes a list of input_ids. The method simply calls the <code>decode</code> method
under the hood.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>input_ids</code>
            </td>
            <td>
                  <code><span title="numpy.typing.NDArray">NDArray</span>[<span title="numpy.int64">int64</span>]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The input_ids to be decoded</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>skip_special_tokens</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Whether to skip the special tokens or not</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="typing.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The decoded texts</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">299</span>
<span class="normal">300</span>
<span class="normal">301</span>
<span class="normal">302</span>
<span class="normal">303</span>
<span class="normal">304</span>
<span class="normal">305</span>
<span class="normal">306</span>
<span class="normal">307</span>
<span class="normal">308</span>
<span class="normal">309</span>
<span class="normal">310</span>
<span class="normal">311</span>
<span class="normal">312</span>
<span class="normal">313</span>
<span class="normal">314</span>
<span class="normal">315</span>
<span class="normal">316</span>
<span class="normal">317</span>
<span class="normal">318</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span>
    <span class="bp">self</span><span class="p">,</span> <span class="n">input_ids</span><span class="p">:</span> <span class="n">npt</span><span class="o">.</span><span class="n">NDArray</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">int64</span><span class="p">],</span> <span class="n">skip_special_tokens</span><span class="p">:</span> <span class="nb">bool</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Decodes a list of input_ids. The method simply calls the `decode` method</span>
<span class="sd">    under the hood.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    input_ids: npt.NDArray[np.int64]</span>
<span class="sd">        The input_ids to be decoded</span>
<span class="sd">    skip_special_tokens: bool</span>
<span class="sd">        Whether to skip the special tokens or not</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    List[str]</span>
<span class="sd">        The decoded texts</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">decode</span><span class="p">(</span><span class="n">input_ids</span><span class="p">,</span> <span class="n">skip_special_tokens</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>



  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">ImagePreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>


        <p>Preprocessor to prepare the <code>deepimage</code> input dataset.</p>
<p>The Preprocessing consists simply on resizing according to their
aspect ratio</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>img_col</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>name of the column with the images filenames</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>img_path</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>path to the dicrectory where the images are stored</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>width</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>width of the resulting processed image.</p>
              </div>
            </td>
            <td>
                  <code>224</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>height</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>width of the resulting processed image.</p>
              </div>
            </td>
            <td>
                  <code>224</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>verbose</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Enable verbose output.</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.aap">aap</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor">AspectAwarePreprocessor</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>an instance of <code>pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor</code></p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.spp">spp</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.image_utils.SimplePreprocessor" href="utils/image_utils.html#pytorch_widedeep.utils.image_utils.SimplePreprocessor">SimplePreprocessor</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>an instance of <code>pytorch_widedeep.utils.image_utils.SimplePreprocessor</code></p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.normalise_metrics">normalise_metrics</span></code></td>
            <td>
                  <code>Dict</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dict containing the normalisation metrics of the image dataset, i.e.
mean and std for the R, G and B channels</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ImagePreprocessor</span>
<span class="gp">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">path_to_image1</span> <span class="o">=</span> <span class="s1">&#39;tests/test_data_utils/images/galaxy1.png&#39;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">path_to_image2</span> <span class="o">=</span> <span class="s1">&#39;tests/test_data_utils/images/galaxy2.png&#39;</span>
<span class="gp">&gt;&gt;&gt;</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df_train</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;images_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">path_to_image1</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df_test</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;images_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="n">path_to_image2</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">img_preprocessor</span> <span class="o">=</span> <span class="n">ImagePreprocessor</span><span class="p">(</span><span class="n">img_col</span><span class="o">=</span><span class="s1">&#39;images_column&#39;</span><span class="p">,</span> <span class="n">img_path</span><span class="o">=</span><span class="s1">&#39;.&#39;</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">resized_images</span> <span class="o">=</span> <span class="n">img_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">new_resized_images</span> <span class="o">=</span> <span class="n">img_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_train</span><span class="p">)</span>
</code></pre></div>
    <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>NOTE</strong>:
Normalising metrics will only be computed when the <code>fit_transform</code>
method is run. Running <code>transform</code> only will not change the computed
metrics and running <code>fit</code> only simply instantiates the resizing
functions.</p>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 15</span>
<span class="normal"> 16</span>
<span class="normal"> 17</span>
<span class="normal"> 18</span>
<span class="normal"> 19</span>
<span class="normal"> 20</span>
<span class="normal"> 21</span>
<span class="normal"> 22</span>
<span class="normal"> 23</span>
<span class="normal"> 24</span>
<span class="normal"> 25</span>
<span class="normal"> 26</span>
<span class="normal"> 27</span>
<span class="normal"> 28</span>
<span class="normal"> 29</span>
<span class="normal"> 30</span>
<span class="normal"> 31</span>
<span class="normal"> 32</span>
<span class="normal"> 33</span>
<span class="normal"> 34</span>
<span class="normal"> 35</span>
<span class="normal"> 36</span>
<span class="normal"> 37</span>
<span class="normal"> 38</span>
<span class="normal"> 39</span>
<span class="normal"> 40</span>
<span class="normal"> 41</span>
<span class="normal"> 42</span>
<span class="normal"> 43</span>
<span class="normal"> 44</span>
<span class="normal"> 45</span>
<span class="normal"> 46</span>
<span class="normal"> 47</span>
<span class="normal"> 48</span>
<span class="normal"> 49</span>
<span class="normal"> 50</span>
<span class="normal"> 51</span>
<span class="normal"> 52</span>
<span class="normal"> 53</span>
<span class="normal"> 54</span>
<span class="normal"> 55</span>
<span class="normal"> 56</span>
<span class="normal"> 57</span>
<span class="normal"> 58</span>
<span class="normal"> 59</span>
<span class="normal"> 60</span>
<span class="normal"> 61</span>
<span class="normal"> 62</span>
<span class="normal"> 63</span>
<span class="normal"> 64</span>
<span class="normal"> 65</span>
<span class="normal"> 66</span>
<span class="normal"> 67</span>
<span class="normal"> 68</span>
<span class="normal"> 69</span>
<span class="normal"> 70</span>
<span class="normal"> 71</span>
<span class="normal"> 72</span>
<span class="normal"> 73</span>
<span class="normal"> 74</span>
<span class="normal"> 75</span>
<span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span>
<span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ImagePreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the ``deepimage`` input dataset.</span>

<span class="sd">    The Preprocessing consists simply on resizing according to their</span>
<span class="sd">    aspect ratio</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    img_col: str</span>
<span class="sd">        name of the column with the images filenames</span>
<span class="sd">    img_path: str</span>
<span class="sd">        path to the dicrectory where the images are stored</span>
<span class="sd">    width: int, default=224</span>
<span class="sd">        width of the resulting processed image.</span>
<span class="sd">    height: int, default=224</span>
<span class="sd">        width of the resulting processed image.</span>
<span class="sd">    verbose: int, default 1</span>
<span class="sd">        Enable verbose output.</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    aap: AspectAwarePreprocessor</span>
<span class="sd">        an instance of `pytorch_widedeep.utils.image_utils.AspectAwarePreprocessor`</span>
<span class="sd">    spp: SimplePreprocessor</span>
<span class="sd">        an instance of `pytorch_widedeep.utils.image_utils.SimplePreprocessor`</span>
<span class="sd">    normalise_metrics: Dict</span>
<span class="sd">        Dict containing the normalisation metrics of the image dataset, i.e.</span>
<span class="sd">        mean and std for the R, G and B channels</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt;</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ImagePreprocessor</span>
<span class="sd">    &gt;&gt;&gt;</span>
<span class="sd">    &gt;&gt;&gt; path_to_image1 = &#39;tests/test_data_utils/images/galaxy1.png&#39;</span>
<span class="sd">    &gt;&gt;&gt; path_to_image2 = &#39;tests/test_data_utils/images/galaxy2.png&#39;</span>
<span class="sd">    &gt;&gt;&gt;</span>
<span class="sd">    &gt;&gt;&gt; df_train = pd.DataFrame({&#39;images_column&#39;: [path_to_image1]})</span>
<span class="sd">    &gt;&gt;&gt; df_test = pd.DataFrame({&#39;images_column&#39;: [path_to_image2]})</span>
<span class="sd">    &gt;&gt;&gt; img_preprocessor = ImagePreprocessor(img_col=&#39;images_column&#39;, img_path=&#39;.&#39;, verbose=0)</span>
<span class="sd">    &gt;&gt;&gt; resized_images = img_preprocessor.fit_transform(df_train)</span>
<span class="sd">    &gt;&gt;&gt; new_resized_images = img_preprocessor.transform(df_train)</span>

<span class="sd">    :information_source: **NOTE**:</span>
<span class="sd">    Normalising metrics will only be computed when the ``fit_transform``</span>
<span class="sd">    method is run. Running ``transform`` only will not change the computed</span>
<span class="sd">    metrics and running ``fit`` only simply instantiates the resizing</span>
<span class="sd">    functions.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">img_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">img_path</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">width</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span>
        <span class="n">height</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">224</span><span class="p">,</span>
        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ImagePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">img_col</span> <span class="o">=</span> <span class="n">img_col</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">img_path</span> <span class="o">=</span> <span class="n">img_path</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">width</span> <span class="o">=</span> <span class="n">width</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">height</span> <span class="o">=</span> <span class="n">height</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">aap</span> <span class="o">=</span> <span class="n">AspectAwarePreprocessor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">spp</span> <span class="o">=</span> <span class="n">SimplePreprocessor</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">width</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">height</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">BasePreprocessor</span><span class="p">:</span>
        <span class="c1"># simply include this method to comply with the BasePreprocessor and</span>
        <span class="c1"># for consistency with the rest of the preprocessors</span>
        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">transform_sample</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">img</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
        <span class="n">aspect_r</span> <span class="o">=</span> <span class="n">img</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">img</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
        <span class="k">if</span> <span class="n">aspect_r</span> <span class="o">!=</span> <span class="mf">1.0</span><span class="p">:</span>
            <span class="n">resized_img</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">aap</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">resized_img</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">spp</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">resized_img</span>

    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Resizes the images to the input height and width.</span>


<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe with the `img_col`</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            Resized images to the input height and width</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">image_list</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">img_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Reading Images from </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">))</span>
        <span class="n">imgs</span> <span class="o">=</span> <span class="p">[</span><span class="n">cv2</span><span class="o">.</span><span class="n">imread</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">,</span> <span class="n">img</span><span class="p">]))</span> <span class="k">for</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">image_list</span><span class="p">]</span>

        <span class="c1"># finding images with different height and width</span>
        <span class="n">aspect</span> <span class="o">=</span> <span class="p">[(</span><span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="k">for</span> <span class="n">im</span> <span class="ow">in</span> <span class="n">imgs</span><span class="p">]</span>
        <span class="n">aspect_r</span> <span class="o">=</span> <span class="p">[</span><span class="n">a</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">a</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">aspect</span><span class="p">]</span>
        <span class="n">diff_idx</span> <span class="o">=</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">r</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">aspect_r</span><span class="p">)</span> <span class="k">if</span> <span class="n">r</span> <span class="o">!=</span> <span class="mf">1.0</span><span class="p">]</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Resizing&quot;</span><span class="p">)</span>
        <span class="n">resized_imgs</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">enumerate</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">total</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">diff_idx</span><span class="p">:</span>
                <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aap</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="c1"># if aspect ratio is 1:1, no need for AspectAwarePreprocessor</span>
                <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">spp</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Computing normalisation metrics&quot;</span><span class="p">)</span>
            <span class="c1"># mean and std deviation will only be computed when the fit method</span>
            <span class="c1"># is called</span>
            <span class="n">mean_R</span><span class="p">,</span> <span class="n">mean_G</span><span class="p">,</span> <span class="n">mean_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
            <span class="n">std_R</span><span class="p">,</span> <span class="n">std_G</span><span class="p">,</span> <span class="n">std_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">rsz_img</span> <span class="ow">in</span> <span class="n">resized_imgs</span><span class="p">:</span>
                <span class="p">(</span><span class="n">mean_b</span><span class="p">,</span> <span class="n">mean_g</span><span class="p">,</span> <span class="n">mean_r</span><span class="p">),</span> <span class="p">(</span><span class="n">std_b</span><span class="p">,</span> <span class="n">std_g</span><span class="p">,</span> <span class="n">std_r</span><span class="p">)</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">meanStdDev</span><span class="p">(</span>
                    <span class="n">rsz_img</span>
                <span class="p">)</span>
                <span class="n">mean_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_r</span><span class="p">)</span>
                <span class="n">mean_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_g</span><span class="p">)</span>
                <span class="n">mean_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_b</span><span class="p">)</span>
                <span class="n">std_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_r</span><span class="p">)</span>
                <span class="n">std_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_g</span><span class="p">)</span>
                <span class="n">std_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_b</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">normalise_metrics</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
                <span class="n">mean</span><span class="o">=</span><span class="p">{</span>
                    <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                    <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                    <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                <span class="p">},</span>
                <span class="n">std</span><span class="o">=</span><span class="p">{</span>
                    <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                    <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                    <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                <span class="p">},</span>
            <span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">resized_imgs</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        df: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        np.ndarray</span>
<span class="sd">            Resized images to the input height and width</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">transformed_image</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
            <span class="s2">&quot;&#39;inverse_transform&#39; method is not implemented for &#39;ImagePreprocessor&#39;&quot;</span>
        <span class="p">)</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;img_col=</span><span class="si">{img_col}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;img_path=</span><span class="si">{img_path}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;width=</span><span class="si">{width}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;height=</span><span class="si">{height}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;ImagePreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">









<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">transform</span>


<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Resizes the images to the input height and width.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe with the <code>img_col</code></p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Resized images to the input height and width</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span>
<span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Resizes the images to the input height and width.</span>


<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe with the `img_col`</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        Resized images to the input height and width</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">image_list</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">img_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Reading Images from </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">))</span>
    <span class="n">imgs</span> <span class="o">=</span> <span class="p">[</span><span class="n">cv2</span><span class="o">.</span><span class="n">imread</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">img_path</span><span class="p">,</span> <span class="n">img</span><span class="p">]))</span> <span class="k">for</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">image_list</span><span class="p">]</span>

    <span class="c1"># finding images with different height and width</span>
    <span class="n">aspect</span> <span class="o">=</span> <span class="p">[(</span><span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">im</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span> <span class="k">for</span> <span class="n">im</span> <span class="ow">in</span> <span class="n">imgs</span><span class="p">]</span>
    <span class="n">aspect_r</span> <span class="o">=</span> <span class="p">[</span><span class="n">a</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">/</span> <span class="n">a</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">a</span> <span class="ow">in</span> <span class="n">aspect</span><span class="p">]</span>
    <span class="n">diff_idx</span> <span class="o">=</span> <span class="p">[</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">r</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">aspect_r</span><span class="p">)</span> <span class="k">if</span> <span class="n">r</span> <span class="o">!=</span> <span class="mf">1.0</span><span class="p">]</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
        <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Resizing&quot;</span><span class="p">)</span>
    <span class="n">resized_imgs</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">img</span> <span class="ow">in</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">enumerate</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">total</span><span class="o">=</span><span class="nb">len</span><span class="p">(</span><span class="n">imgs</span><span class="p">),</span> <span class="n">disable</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">diff_idx</span><span class="p">:</span>
            <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">aap</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="c1"># if aspect ratio is 1:1, no need for AspectAwarePreprocessor</span>
            <span class="n">resized_imgs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">spp</span><span class="o">.</span><span class="n">preprocess</span><span class="p">(</span><span class="n">img</span><span class="p">))</span>

    <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span><span class="p">:</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
            <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Computing normalisation metrics&quot;</span><span class="p">)</span>
        <span class="c1"># mean and std deviation will only be computed when the fit method</span>
        <span class="c1"># is called</span>
        <span class="n">mean_R</span><span class="p">,</span> <span class="n">mean_G</span><span class="p">,</span> <span class="n">mean_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
        <span class="n">std_R</span><span class="p">,</span> <span class="n">std_G</span><span class="p">,</span> <span class="n">std_B</span> <span class="o">=</span> <span class="p">[],</span> <span class="p">[],</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">rsz_img</span> <span class="ow">in</span> <span class="n">resized_imgs</span><span class="p">:</span>
            <span class="p">(</span><span class="n">mean_b</span><span class="p">,</span> <span class="n">mean_g</span><span class="p">,</span> <span class="n">mean_r</span><span class="p">),</span> <span class="p">(</span><span class="n">std_b</span><span class="p">,</span> <span class="n">std_g</span><span class="p">,</span> <span class="n">std_r</span><span class="p">)</span> <span class="o">=</span> <span class="n">cv2</span><span class="o">.</span><span class="n">meanStdDev</span><span class="p">(</span>
                <span class="n">rsz_img</span>
            <span class="p">)</span>
            <span class="n">mean_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_r</span><span class="p">)</span>
            <span class="n">mean_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_g</span><span class="p">)</span>
            <span class="n">mean_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">mean_b</span><span class="p">)</span>
            <span class="n">std_R</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_r</span><span class="p">)</span>
            <span class="n">std_G</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_g</span><span class="p">)</span>
            <span class="n">std_B</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">std_b</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">normalise_metrics</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span>
            <span class="n">mean</span><span class="o">=</span><span class="p">{</span>
                <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">mean_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
            <span class="p">},</span>
            <span class="n">std</span><span class="o">=</span><span class="p">{</span>
                <span class="s2">&quot;R&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_R</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                <span class="s2">&quot;G&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_G</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
                <span class="s2">&quot;B&quot;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">std_B</span><span class="p">)</span> <span class="o">/</span> <span class="mf">255.0</span><span class="p">,</span>
            <span class="p">},</span>
        <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">compute_normalising_computed</span> <span class="o">=</span> <span class="kc">True</span>
    <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">asarray</span><span class="p">(</span><span class="n">resized_imgs</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit_transform</span>


<a href="#pytorch_widedeep.preprocessing.image_preprocessor.ImagePreprocessor.fit_transform" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Combines <code>fit</code> and <code>transform</code></p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>df</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Resized images to the input height and width</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/image_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Combines `fit` and `transform`</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    df: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    np.ndarray</span>
<span class="sd">        Resized images to the input height and width</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>



  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">DINPreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><span title="pytorch_widedeep.preprocessing.base_preprocessor.BasePreprocessor">BasePreprocessor</span></code></p>


        <p>Preprocessor for Deep Interest Network (DIN) models.</p>
<p>This preprocessor handles the preparation of data for DIN models,
including sequence building, label encoding, and handling of various
types of input columns (categorical, continuous, and sequential).</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>user_id_col</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Name of the column containing user IDs.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>item_embed_col</code>
            </td>
            <td>
                  <code><span title="typing.Union">Union</span>[str, <span title="typing.Tuple">Tuple</span>[str, int]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Name of the column containing item IDs to be embedded, or a tuple of
(column_name, embedding_dim). If embedding_dim is not provided, it
will be automatically calculated using a rule of thumb specified by
<code>embedding_rule</code>. Aliased as <code>item_id_col</code>.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>target_col</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Name of the column containing the target variable.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>max_seq_length</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Maximum length of sequences to be created.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>action_col</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Name of the column containing user actions (if applicable), for
example a rating, or purchased/not-purchased, etc). This 'action_col'
can also be the same as the 'target_col'.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>other_seq_embed_cols</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str] | <span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, int]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of other columns to be treated as sequences. Each element in the
list can be a string with the name of the column or a tuple with the
name of the column and the embedding dimension.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>cat_embed_cols</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[<span title="typing.List">List</span>[str], <span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, int]]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>This and the following parameters are identical to those used in the
TabPreprocessor.<br>
List containing the name of the categorical columns that will be
represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
a Tuple with the name and the embedding dimension (e.g.: <em>[
('education',32), ('relationship',16), ...]</em>)</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>continuous_cols</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the name of the continuous cols</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>quantization_setup</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[int, <span title="typing.Dict">Dict</span>[str, <span title="typing.Union">Union</span>[int, <span title="typing.List">List</span>[float]]]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Continuous columns can be turned into categorical via <code>pd.cut</code>. If
<code>quantization_setup</code> is an <code>int</code>, all continuous columns will be
quantized using this value as the number of bins. Alternatively, a
dictionary where the keys are the column names to quantize and the
values are the either integers indicating the number of bins or a
list of scalars indicating the bin edges can also be used.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>cols_to_scale</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Union">Union</span>[<span title="typing.List">List</span>[str], str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the names of the columns that will be standarised via
sklearn's <code>StandardScaler</code>. It can also be the string <code>'all'</code> in
which case all the continuous cols will be scaled.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>auto_embed_dim</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating whether the embedding dimensions will be
automatically defined via rule of thumb. See <code>embedding_rule</code>
below.</p>
              </div>
            </td>
            <td>
                  <code>True</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>embedding_rule</code>
            </td>
            <td>
                  <code><span title="typing.Literal">Literal</span>[&#39;google&#39;, &#39;fastai_old&#39;, &#39;fastai_new&#39;]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>If <code>auto_embed_dim=True</code>, this is the choice of embedding rule of
thumb. Choices are:</p>
<ul>
<li>
<p><em>fastai_new</em>: <span class="arithmatex">\(min(600, round(1.6     imes n_{cat}^{0.56}))\)</span></p>
</li>
<li>
<p><em>fastai_old</em>: <span class="arithmatex">\(min(50, (n_{cat}//{2})+1)\)</span></p>
</li>
<li>
<p><em>google</em>: <span class="arithmatex">\(min(600, round(n_{cat}^{0.24}))\)</span></p>
</li>
</ul>
              </div>
            </td>
            <td>
                  <code>&#39;fastai_new&#39;</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>default_embed_dim</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dimension for the embeddings if the embedding dimension is not
provided in the <code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is
set to <code>False</code>.</p>
              </div>
            </td>
            <td>
                  <code>16</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>verbose</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Verbosity level.</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>scale</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
 releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
 whether or not to scale/standarise continuous cols. It is important
 to emphasize that all the DL models for tabular data in the library
 also include the possibility of normalising the input continuous
 features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
 <code>scale_cont_cols</code>.</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>already_standard</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
 releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
 name of the continuous cols that do not need to be
 scaled/standarised.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Other Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code>**kwargs</code></td>
            <td>
            </td>
            <td>
              <div class="doc-md-description">
                <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.is_fitted">is_fitted</span></code></td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Whether the preprocessor has been fitted.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.has_standard_tab_data">has_standard_tab_data</span></code></td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Whether the data includes standard tabular data.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.tab_preprocessor">tab_preprocessor</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Preprocessor for standard tabular data (if applicable).</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.din_columns_idx">din_columns_idx</span></code></td>
            <td>
                  <code><span title="typing.Dict">Dict</span>[str, int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary mapping column names to their indices in the processed data.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.item_le">item_le</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Label encoder for item IDs.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.n_items">n_items</span></code></td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Number of unique items.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.user_behaviour_config">user_behaviour_config</span></code></td>
            <td>
                  <code><span title="typing.Tuple">Tuple</span>[<span title="typing.List">List</span>[str], int, int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Configuration for user behavior sequences.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.action_le">action_le</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Label encoder for action column (if applicable).</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.n_actions">n_actions</span></code></td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Number of unique actions (if applicable).</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.action_seq_config">action_seq_config</span></code></td>
            <td>
                  <code><span title="typing.Tuple">Tuple</span>[<span title="typing.List">List</span>[str], int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Configuration for action sequences (if applicable).</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.other_seq_le">other_seq_le</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Label encoder for other sequence columns (if applicable).</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.n_other_seq_cols">n_other_seq_cols</span></code></td>
            <td>
                  <code><span title="typing.Dict">Dict</span>[str, int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Number of unique values in each other sequence column.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.din_preprocessor.DINPreprocessor.other_seq_config">other_seq_config</span></code></td>
            <td>
                  <code><span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[<span title="typing.List">List</span>[str], int, int]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Configuration for other sequence columns.</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">DINPreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">data</span> <span class="o">=</span> <span class="p">{</span>
<span class="gp">... </span>   <span class="s1">&#39;user_id&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
<span class="gp">... </span>   <span class="s1">&#39;item_id&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">101</span><span class="p">,</span> <span class="mi">102</span><span class="p">,</span> <span class="mi">103</span><span class="p">,</span> <span class="mi">101</span><span class="p">,</span> <span class="mi">103</span><span class="p">,</span> <span class="mi">104</span><span class="p">,</span> <span class="mi">102</span><span class="p">,</span> <span class="mi">103</span><span class="p">,</span> <span class="mi">104</span><span class="p">],</span>
<span class="gp">... </span>   <span class="s1">&#39;timestamp&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
<span class="gp">... </span>   <span class="s1">&#39;category&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;C&#39;</span><span class="p">,</span> <span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;C&#39;</span><span class="p">],</span>
<span class="gp">... </span>   <span class="s1">&#39;price&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mf">10.5</span><span class="p">,</span> <span class="mf">15.0</span><span class="p">,</span> <span class="mf">12.0</span><span class="p">,</span> <span class="mf">10.5</span><span class="p">,</span> <span class="mf">12.0</span><span class="p">,</span> <span class="mf">20.0</span><span class="p">,</span> <span class="mf">15.0</span><span class="p">,</span> <span class="mf">12.0</span><span class="p">,</span> <span class="mf">20.0</span><span class="p">],</span>
<span class="gp">... </span>   <span class="s1">&#39;rating&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">... </span><span class="p">}</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">din_preprocessor</span> <span class="o">=</span> <span class="n">DINPreprocessor</span><span class="p">(</span>
<span class="gp">... </span>    <span class="n">user_id_col</span><span class="o">=</span><span class="s1">&#39;user_id&#39;</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">item_embed_col</span><span class="o">=</span><span class="s1">&#39;item_id&#39;</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">target_col</span><span class="o">=</span><span class="s1">&#39;rating&#39;</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">max_seq_length</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">action_col</span><span class="o">=</span><span class="s1">&#39;rating&#39;</span><span class="p">,</span>
<span class="gp">... </span>    <span class="n">other_seq_embed_cols</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;category&#39;</span><span class="p">],</span>
<span class="gp">... </span>    <span class="n">cat_embed_cols</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;user_id&#39;</span><span class="p">],</span>
<span class="gp">... </span>    <span class="n">continuous_cols</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;price&#39;</span><span class="p">],</span>
<span class="gp">... </span>    <span class="n">cols_to_scale</span><span class="o">=</span><span class="p">[</span><span class="s1">&#39;price&#39;</span><span class="p">]</span>
<span class="gp">... </span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">din_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/din_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 21</span>
<span class="normal"> 22</span>
<span class="normal"> 23</span>
<span class="normal"> 24</span>
<span class="normal"> 25</span>
<span class="normal"> 26</span>
<span class="normal"> 27</span>
<span class="normal"> 28</span>
<span class="normal"> 29</span>
<span class="normal"> 30</span>
<span class="normal"> 31</span>
<span class="normal"> 32</span>
<span class="normal"> 33</span>
<span class="normal"> 34</span>
<span class="normal"> 35</span>
<span class="normal"> 36</span>
<span class="normal"> 37</span>
<span class="normal"> 38</span>
<span class="normal"> 39</span>
<span class="normal"> 40</span>
<span class="normal"> 41</span>
<span class="normal"> 42</span>
<span class="normal"> 43</span>
<span class="normal"> 44</span>
<span class="normal"> 45</span>
<span class="normal"> 46</span>
<span class="normal"> 47</span>
<span class="normal"> 48</span>
<span class="normal"> 49</span>
<span class="normal"> 50</span>
<span class="normal"> 51</span>
<span class="normal"> 52</span>
<span class="normal"> 53</span>
<span class="normal"> 54</span>
<span class="normal"> 55</span>
<span class="normal"> 56</span>
<span class="normal"> 57</span>
<span class="normal"> 58</span>
<span class="normal"> 59</span>
<span class="normal"> 60</span>
<span class="normal"> 61</span>
<span class="normal"> 62</span>
<span class="normal"> 63</span>
<span class="normal"> 64</span>
<span class="normal"> 65</span>
<span class="normal"> 66</span>
<span class="normal"> 67</span>
<span class="normal"> 68</span>
<span class="normal"> 69</span>
<span class="normal"> 70</span>
<span class="normal"> 71</span>
<span class="normal"> 72</span>
<span class="normal"> 73</span>
<span class="normal"> 74</span>
<span class="normal"> 75</span>
<span class="normal"> 76</span>
<span class="normal"> 77</span>
<span class="normal"> 78</span>
<span class="normal"> 79</span>
<span class="normal"> 80</span>
<span class="normal"> 81</span>
<span class="normal"> 82</span>
<span class="normal"> 83</span>
<span class="normal"> 84</span>
<span class="normal"> 85</span>
<span class="normal"> 86</span>
<span class="normal"> 87</span>
<span class="normal"> 88</span>
<span class="normal"> 89</span>
<span class="normal"> 90</span>
<span class="normal"> 91</span>
<span class="normal"> 92</span>
<span class="normal"> 93</span>
<span class="normal"> 94</span>
<span class="normal"> 95</span>
<span class="normal"> 96</span>
<span class="normal"> 97</span>
<span class="normal"> 98</span>
<span class="normal"> 99</span>
<span class="normal">100</span>
<span class="normal">101</span>
<span class="normal">102</span>
<span class="normal">103</span>
<span class="normal">104</span>
<span class="normal">105</span>
<span class="normal">106</span>
<span class="normal">107</span>
<span class="normal">108</span>
<span class="normal">109</span>
<span class="normal">110</span>
<span class="normal">111</span>
<span class="normal">112</span>
<span class="normal">113</span>
<span class="normal">114</span>
<span class="normal">115</span>
<span class="normal">116</span>
<span class="normal">117</span>
<span class="normal">118</span>
<span class="normal">119</span>
<span class="normal">120</span>
<span class="normal">121</span>
<span class="normal">122</span>
<span class="normal">123</span>
<span class="normal">124</span>
<span class="normal">125</span>
<span class="normal">126</span>
<span class="normal">127</span>
<span class="normal">128</span>
<span class="normal">129</span>
<span class="normal">130</span>
<span class="normal">131</span>
<span class="normal">132</span>
<span class="normal">133</span>
<span class="normal">134</span>
<span class="normal">135</span>
<span class="normal">136</span>
<span class="normal">137</span>
<span class="normal">138</span>
<span class="normal">139</span>
<span class="normal">140</span>
<span class="normal">141</span>
<span class="normal">142</span>
<span class="normal">143</span>
<span class="normal">144</span>
<span class="normal">145</span>
<span class="normal">146</span>
<span class="normal">147</span>
<span class="normal">148</span>
<span class="normal">149</span>
<span class="normal">150</span>
<span class="normal">151</span>
<span class="normal">152</span>
<span class="normal">153</span>
<span class="normal">154</span>
<span class="normal">155</span>
<span class="normal">156</span>
<span class="normal">157</span>
<span class="normal">158</span>
<span class="normal">159</span>
<span class="normal">160</span>
<span class="normal">161</span>
<span class="normal">162</span>
<span class="normal">163</span>
<span class="normal">164</span>
<span class="normal">165</span>
<span class="normal">166</span>
<span class="normal">167</span>
<span class="normal">168</span>
<span class="normal">169</span>
<span class="normal">170</span>
<span class="normal">171</span>
<span class="normal">172</span>
<span class="normal">173</span>
<span class="normal">174</span>
<span class="normal">175</span>
<span class="normal">176</span>
<span class="normal">177</span>
<span class="normal">178</span>
<span class="normal">179</span>
<span class="normal">180</span>
<span class="normal">181</span>
<span class="normal">182</span>
<span class="normal">183</span>
<span class="normal">184</span>
<span class="normal">185</span>
<span class="normal">186</span>
<span class="normal">187</span>
<span class="normal">188</span>
<span class="normal">189</span>
<span class="normal">190</span>
<span class="normal">191</span>
<span class="normal">192</span>
<span class="normal">193</span>
<span class="normal">194</span>
<span class="normal">195</span>
<span class="normal">196</span>
<span class="normal">197</span>
<span class="normal">198</span>
<span class="normal">199</span>
<span class="normal">200</span>
<span class="normal">201</span>
<span class="normal">202</span>
<span class="normal">203</span>
<span class="normal">204</span>
<span class="normal">205</span>
<span class="normal">206</span>
<span class="normal">207</span>
<span class="normal">208</span>
<span class="normal">209</span>
<span class="normal">210</span>
<span class="normal">211</span>
<span class="normal">212</span>
<span class="normal">213</span>
<span class="normal">214</span>
<span class="normal">215</span>
<span class="normal">216</span>
<span class="normal">217</span>
<span class="normal">218</span>
<span class="normal">219</span>
<span class="normal">220</span>
<span class="normal">221</span>
<span class="normal">222</span>
<span class="normal">223</span>
<span class="normal">224</span>
<span class="normal">225</span>
<span class="normal">226</span>
<span class="normal">227</span>
<span class="normal">228</span>
<span class="normal">229</span>
<span class="normal">230</span>
<span class="normal">231</span>
<span class="normal">232</span>
<span class="normal">233</span>
<span class="normal">234</span>
<span class="normal">235</span>
<span class="normal">236</span>
<span class="normal">237</span>
<span class="normal">238</span>
<span class="normal">239</span>
<span class="normal">240</span>
<span class="normal">241</span>
<span class="normal">242</span>
<span class="normal">243</span>
<span class="normal">244</span>
<span class="normal">245</span>
<span class="normal">246</span>
<span class="normal">247</span>
<span class="normal">248</span>
<span class="normal">249</span>
<span class="normal">250</span>
<span class="normal">251</span>
<span class="normal">252</span>
<span class="normal">253</span>
<span class="normal">254</span>
<span class="normal">255</span>
<span class="normal">256</span>
<span class="normal">257</span>
<span class="normal">258</span>
<span class="normal">259</span>
<span class="normal">260</span>
<span class="normal">261</span>
<span class="normal">262</span>
<span class="normal">263</span>
<span class="normal">264</span>
<span class="normal">265</span>
<span class="normal">266</span>
<span class="normal">267</span>
<span class="normal">268</span>
<span class="normal">269</span>
<span class="normal">270</span>
<span class="normal">271</span>
<span class="normal">272</span>
<span class="normal">273</span>
<span class="normal">274</span>
<span class="normal">275</span>
<span class="normal">276</span>
<span class="normal">277</span>
<span class="normal">278</span>
<span class="normal">279</span>
<span class="normal">280</span>
<span class="normal">281</span>
<span class="normal">282</span>
<span class="normal">283</span>
<span class="normal">284</span>
<span class="normal">285</span>
<span class="normal">286</span>
<span class="normal">287</span>
<span class="normal">288</span>
<span class="normal">289</span>
<span class="normal">290</span>
<span class="normal">291</span>
<span class="normal">292</span>
<span class="normal">293</span>
<span class="normal">294</span>
<span class="normal">295</span>
<span class="normal">296</span>
<span class="normal">297</span>
<span class="normal">298</span>
<span class="normal">299</span>
<span class="normal">300</span>
<span class="normal">301</span>
<span class="normal">302</span>
<span class="normal">303</span>
<span class="normal">304</span>
<span class="normal">305</span>
<span class="normal">306</span>
<span class="normal">307</span>
<span class="normal">308</span>
<span class="normal">309</span>
<span class="normal">310</span>
<span class="normal">311</span>
<span class="normal">312</span>
<span class="normal">313</span>
<span class="normal">314</span>
<span class="normal">315</span>
<span class="normal">316</span>
<span class="normal">317</span>
<span class="normal">318</span>
<span class="normal">319</span>
<span class="normal">320</span>
<span class="normal">321</span>
<span class="normal">322</span>
<span class="normal">323</span>
<span class="normal">324</span>
<span class="normal">325</span>
<span class="normal">326</span>
<span class="normal">327</span>
<span class="normal">328</span>
<span class="normal">329</span>
<span class="normal">330</span>
<span class="normal">331</span>
<span class="normal">332</span>
<span class="normal">333</span>
<span class="normal">334</span>
<span class="normal">335</span>
<span class="normal">336</span>
<span class="normal">337</span>
<span class="normal">338</span>
<span class="normal">339</span>
<span class="normal">340</span>
<span class="normal">341</span>
<span class="normal">342</span>
<span class="normal">343</span>
<span class="normal">344</span>
<span class="normal">345</span>
<span class="normal">346</span>
<span class="normal">347</span>
<span class="normal">348</span>
<span class="normal">349</span>
<span class="normal">350</span>
<span class="normal">351</span>
<span class="normal">352</span>
<span class="normal">353</span>
<span class="normal">354</span>
<span class="normal">355</span>
<span class="normal">356</span>
<span class="normal">357</span>
<span class="normal">358</span>
<span class="normal">359</span>
<span class="normal">360</span>
<span class="normal">361</span>
<span class="normal">362</span>
<span class="normal">363</span>
<span class="normal">364</span>
<span class="normal">365</span>
<span class="normal">366</span>
<span class="normal">367</span>
<span class="normal">368</span>
<span class="normal">369</span>
<span class="normal">370</span>
<span class="normal">371</span>
<span class="normal">372</span>
<span class="normal">373</span>
<span class="normal">374</span>
<span class="normal">375</span>
<span class="normal">376</span>
<span class="normal">377</span>
<span class="normal">378</span>
<span class="normal">379</span>
<span class="normal">380</span>
<span class="normal">381</span>
<span class="normal">382</span>
<span class="normal">383</span>
<span class="normal">384</span>
<span class="normal">385</span>
<span class="normal">386</span>
<span class="normal">387</span>
<span class="normal">388</span>
<span class="normal">389</span>
<span class="normal">390</span>
<span class="normal">391</span>
<span class="normal">392</span>
<span class="normal">393</span>
<span class="normal">394</span>
<span class="normal">395</span>
<span class="normal">396</span>
<span class="normal">397</span>
<span class="normal">398</span>
<span class="normal">399</span>
<span class="normal">400</span>
<span class="normal">401</span>
<span class="normal">402</span>
<span class="normal">403</span>
<span class="normal">404</span>
<span class="normal">405</span>
<span class="normal">406</span>
<span class="normal">407</span>
<span class="normal">408</span>
<span class="normal">409</span>
<span class="normal">410</span>
<span class="normal">411</span>
<span class="normal">412</span>
<span class="normal">413</span>
<span class="normal">414</span>
<span class="normal">415</span>
<span class="normal">416</span>
<span class="normal">417</span>
<span class="normal">418</span>
<span class="normal">419</span>
<span class="normal">420</span>
<span class="normal">421</span>
<span class="normal">422</span>
<span class="normal">423</span>
<span class="normal">424</span>
<span class="normal">425</span>
<span class="normal">426</span>
<span class="normal">427</span>
<span class="normal">428</span>
<span class="normal">429</span>
<span class="normal">430</span>
<span class="normal">431</span>
<span class="normal">432</span>
<span class="normal">433</span>
<span class="normal">434</span>
<span class="normal">435</span>
<span class="normal">436</span>
<span class="normal">437</span>
<span class="normal">438</span>
<span class="normal">439</span>
<span class="normal">440</span>
<span class="normal">441</span>
<span class="normal">442</span>
<span class="normal">443</span>
<span class="normal">444</span>
<span class="normal">445</span>
<span class="normal">446</span>
<span class="normal">447</span>
<span class="normal">448</span>
<span class="normal">449</span>
<span class="normal">450</span>
<span class="normal">451</span>
<span class="normal">452</span>
<span class="normal">453</span>
<span class="normal">454</span>
<span class="normal">455</span>
<span class="normal">456</span>
<span class="normal">457</span>
<span class="normal">458</span>
<span class="normal">459</span>
<span class="normal">460</span>
<span class="normal">461</span>
<span class="normal">462</span>
<span class="normal">463</span>
<span class="normal">464</span>
<span class="normal">465</span>
<span class="normal">466</span>
<span class="normal">467</span>
<span class="normal">468</span>
<span class="normal">469</span>
<span class="normal">470</span>
<span class="normal">471</span>
<span class="normal">472</span>
<span class="normal">473</span>
<span class="normal">474</span>
<span class="normal">475</span>
<span class="normal">476</span>
<span class="normal">477</span>
<span class="normal">478</span>
<span class="normal">479</span>
<span class="normal">480</span>
<span class="normal">481</span>
<span class="normal">482</span>
<span class="normal">483</span>
<span class="normal">484</span>
<span class="normal">485</span>
<span class="normal">486</span>
<span class="normal">487</span>
<span class="normal">488</span>
<span class="normal">489</span>
<span class="normal">490</span>
<span class="normal">491</span>
<span class="normal">492</span>
<span class="normal">493</span>
<span class="normal">494</span>
<span class="normal">495</span>
<span class="normal">496</span>
<span class="normal">497</span>
<span class="normal">498</span>
<span class="normal">499</span>
<span class="normal">500</span>
<span class="normal">501</span>
<span class="normal">502</span>
<span class="normal">503</span>
<span class="normal">504</span>
<span class="normal">505</span>
<span class="normal">506</span>
<span class="normal">507</span>
<span class="normal">508</span>
<span class="normal">509</span>
<span class="normal">510</span>
<span class="normal">511</span>
<span class="normal">512</span>
<span class="normal">513</span>
<span class="normal">514</span>
<span class="normal">515</span>
<span class="normal">516</span>
<span class="normal">517</span>
<span class="normal">518</span>
<span class="normal">519</span>
<span class="normal">520</span>
<span class="normal">521</span>
<span class="normal">522</span>
<span class="normal">523</span>
<span class="normal">524</span>
<span class="normal">525</span>
<span class="normal">526</span>
<span class="normal">527</span>
<span class="normal">528</span>
<span class="normal">529</span>
<span class="normal">530</span>
<span class="normal">531</span>
<span class="normal">532</span>
<span class="normal">533</span>
<span class="normal">534</span>
<span class="normal">535</span>
<span class="normal">536</span>
<span class="normal">537</span>
<span class="normal">538</span>
<span class="normal">539</span>
<span class="normal">540</span>
<span class="normal">541</span>
<span class="normal">542</span>
<span class="normal">543</span>
<span class="normal">544</span>
<span class="normal">545</span>
<span class="normal">546</span>
<span class="normal">547</span>
<span class="normal">548</span>
<span class="normal">549</span>
<span class="normal">550</span>
<span class="normal">551</span>
<span class="normal">552</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">DINPreprocessor</span><span class="p">(</span><span class="n">BasePreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Preprocessor for Deep Interest Network (DIN) models.</span>

<span class="sd">    This preprocessor handles the preparation of data for DIN models,</span>
<span class="sd">    including sequence building, label encoding, and handling of various</span>
<span class="sd">    types of input columns (categorical, continuous, and sequential).</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    user_id_col : str</span>
<span class="sd">        Name of the column containing user IDs.</span>
<span class="sd">    item_embed_col : Union[str, Tuple[str, int]]</span>
<span class="sd">        Name of the column containing item IDs to be embedded, or a tuple of</span>
<span class="sd">        (column_name, embedding_dim). If embedding_dim is not provided, it</span>
<span class="sd">        will be automatically calculated using a rule of thumb specified by</span>
<span class="sd">        `embedding_rule`. Aliased as `item_id_col`.</span>
<span class="sd">    target_col : str</span>
<span class="sd">        Name of the column containing the target variable.</span>
<span class="sd">    max_seq_length : int</span>
<span class="sd">        Maximum length of sequences to be created.</span>
<span class="sd">    action_col : Optional[str], default=None</span>
<span class="sd">        Name of the column containing user actions (if applicable), for</span>
<span class="sd">        example a rating, or purchased/not-purchased, etc). This &#39;action_col&#39;</span>
<span class="sd">        can also be the same as the &#39;target_col&#39;.</span>
<span class="sd">    other_seq_embed_cols : Optional[List[str] | List[Tuple[str, int]]], default=None</span>
<span class="sd">        List of other columns to be treated as sequences. Each element in the</span>
<span class="sd">        list can be a string with the name of the column or a tuple with the</span>
<span class="sd">        name of the column and the embedding dimension.</span>

<span class="sd">    cat_embed_cols : Optional[Union[List[str], List[Tuple[str, int]]]], default=None</span>
<span class="sd">        This and the following parameters are identical to those used in the</span>
<span class="sd">        TabPreprocessor.&lt;br&gt;</span>
<span class="sd">        List containing the name of the categorical columns that will be</span>
<span class="sd">        represented by embeddings (e.g. _[&#39;education&#39;, &#39;relationship&#39;, ...]_) or</span>
<span class="sd">        a Tuple with the name and the embedding dimension (e.g.: _[</span>
<span class="sd">        (&#39;education&#39;,32), (&#39;relationship&#39;,16), ...]_)</span>
<span class="sd">    continuous_cols: List, default = None</span>
<span class="sd">        List with the name of the continuous cols</span>
<span class="sd">    quantization_setup: int or Dict, default = None</span>
<span class="sd">        Continuous columns can be turned into categorical via `pd.cut`. If</span>
<span class="sd">        `quantization_setup` is an `int`, all continuous columns will be</span>
<span class="sd">        quantized using this value as the number of bins. Alternatively, a</span>
<span class="sd">        dictionary where the keys are the column names to quantize and the</span>
<span class="sd">        values are the either integers indicating the number of bins or a</span>
<span class="sd">        list of scalars indicating the bin edges can also be used.</span>
<span class="sd">    cols_to_scale: List or str, default = None,</span>
<span class="sd">        List with the names of the columns that will be standarised via</span>
<span class="sd">        sklearn&#39;s `StandardScaler`. It can also be the string `&#39;all&#39;` in</span>
<span class="sd">        which case all the continuous cols will be scaled.</span>
<span class="sd">    auto_embed_dim: bool, default = True</span>
<span class="sd">        Boolean indicating whether the embedding dimensions will be</span>
<span class="sd">        automatically defined via rule of thumb. See `embedding_rule`</span>
<span class="sd">        below.</span>
<span class="sd">    embedding_rule: str, default = &#39;fastai_new&#39;</span>
<span class="sd">        If `auto_embed_dim=True`, this is the choice of embedding rule of</span>
<span class="sd">        thumb. Choices are:</span>

<span class="sd">        - _fastai_new_: $min(600, round(1.6 \times n_{cat}^{0.56}))$</span>

<span class="sd">        - _fastai_old_: $min(50, (n_{cat}//{2})+1)$</span>

<span class="sd">        - _google_: $min(600, round(n_{cat}^{0.24}))$</span>
<span class="sd">    default_embed_dim: int, default=16</span>
<span class="sd">        Dimension for the embeddings if the embedding dimension is not</span>
<span class="sd">        provided in the `cat_embed_cols` parameter and `auto_embed_dim` is</span>
<span class="sd">        set to `False`.</span>
<span class="sd">    verbose : int, default=1</span>
<span class="sd">        Verbosity level.</span>
<span class="sd">    scale: bool, default = False</span>
<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating</span>
<span class="sd">         whether or not to scale/standarise continuous cols. It is important</span>
<span class="sd">         to emphasize that all the DL models for tabular data in the library</span>
<span class="sd">         also include the possibility of normalising the input continuous</span>
<span class="sd">         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:</span>
<span class="sd">         `scale_cont_cols`.</span>
<span class="sd">    already_standard: List, default = None</span>
<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the</span>
<span class="sd">         name of the continuous cols that do not need to be</span>
<span class="sd">         scaled/standarised.</span>

<span class="sd">    Other Parameters</span>
<span class="sd">    ----------------</span>
<span class="sd">    **kwargs: dict</span>
<span class="sd">        `pd.cut` and `StandardScaler` related args</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    is_fitted : bool</span>
<span class="sd">        Whether the preprocessor has been fitted.</span>
<span class="sd">    has_standard_tab_data : bool</span>
<span class="sd">        Whether the data includes standard tabular data.</span>
<span class="sd">    tab_preprocessor : TabPreprocessor</span>
<span class="sd">        Preprocessor for standard tabular data (if applicable).</span>
<span class="sd">    din_columns_idx : Dict[str, int]</span>
<span class="sd">        Dictionary mapping column names to their indices in the processed data.</span>
<span class="sd">    item_le : LabelEncoder</span>
<span class="sd">        Label encoder for item IDs.</span>
<span class="sd">    n_items : int</span>
<span class="sd">        Number of unique items.</span>
<span class="sd">    user_behaviour_config : Tuple[List[str], int, int]</span>
<span class="sd">        Configuration for user behavior sequences.</span>
<span class="sd">    action_le : LabelEncoder</span>
<span class="sd">        Label encoder for action column (if applicable).</span>
<span class="sd">    n_actions : int</span>
<span class="sd">        Number of unique actions (if applicable).</span>
<span class="sd">    action_seq_config : Tuple[List[str], int]</span>
<span class="sd">        Configuration for action sequences (if applicable).</span>
<span class="sd">    other_seq_le : LabelEncoder</span>
<span class="sd">        Label encoder for other sequence columns (if applicable).</span>
<span class="sd">    n_other_seq_cols : Dict[str, int]</span>
<span class="sd">        Number of unique values in each other sequence column.</span>
<span class="sd">    other_seq_config : List[Tuple[List[str], int, int]]</span>
<span class="sd">        Configuration for other sequence columns.</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import DINPreprocessor</span>
<span class="sd">    &gt;&gt;&gt; data = {</span>
<span class="sd">    ...    &#39;user_id&#39;: [1, 1, 1, 2, 2, 2, 3, 3, 3],</span>
<span class="sd">    ...    &#39;item_id&#39;: [101, 102, 103, 101, 103, 104, 102, 103, 104],</span>
<span class="sd">    ...    &#39;timestamp&#39;: [1, 2, 3, 1, 2, 3, 1, 2, 3],</span>
<span class="sd">    ...    &#39;category&#39;: [&#39;A&#39;, &#39;B&#39;, &#39;A&#39;, &#39;B&#39;, &#39;A&#39;, &#39;C&#39;, &#39;B&#39;, &#39;A&#39;, &#39;C&#39;],</span>
<span class="sd">    ...    &#39;price&#39;: [10.5, 15.0, 12.0, 10.5, 12.0, 20.0, 15.0, 12.0, 20.0],</span>
<span class="sd">    ...    &#39;rating&#39;: [0, 1, 0, 1, 0, 1, 0, 1, 0]</span>
<span class="sd">    ... }</span>
<span class="sd">    &gt;&gt;&gt; df = pd.DataFrame(data)</span>
<span class="sd">    &gt;&gt;&gt; din_preprocessor = DINPreprocessor(</span>
<span class="sd">    ...     user_id_col=&#39;user_id&#39;,</span>
<span class="sd">    ...     item_embed_col=&#39;item_id&#39;,</span>
<span class="sd">    ...     target_col=&#39;rating&#39;,</span>
<span class="sd">    ...     max_seq_length=2,</span>
<span class="sd">    ...     action_col=&#39;rating&#39;,</span>
<span class="sd">    ...     other_seq_embed_cols=[&#39;category&#39;],</span>
<span class="sd">    ...     cat_embed_cols=[&#39;user_id&#39;],</span>
<span class="sd">    ...     continuous_cols=[&#39;price&#39;],</span>
<span class="sd">    ...     cols_to_scale=[&#39;price&#39;]</span>
<span class="sd">    ... )</span>
<span class="sd">    &gt;&gt;&gt; X, y = din_preprocessor.fit_transform(df)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;item_embed_col&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;item_id_col&quot;</span><span class="p">])</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="o">*</span><span class="p">,</span>
        <span class="n">user_id_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">item_embed_col</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span>
        <span class="n">target_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">max_seq_length</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">action_col</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">other_seq_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">quantization_setup</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span>
            <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]]</span>
        <span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">auto_embed_dim</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
        <span class="n">embedding_rule</span><span class="p">:</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">&quot;google&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_old&quot;</span><span class="p">,</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;fastai_new&quot;</span><span class="p">,</span>
        <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
        <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span> <span class="o">=</span> <span class="n">user_id_col</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span> <span class="o">=</span> <span class="n">item_embed_col</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">=</span> <span class="n">max_seq_length</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span> <span class="o">=</span> <span class="n">target_col</span> <span class="k">if</span> <span class="n">target_col</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="s2">&quot;target&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span> <span class="o">=</span> <span class="n">action_col</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span> <span class="o">=</span> <span class="n">other_seq_embed_cols</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="n">cat_embed_cols</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="o">=</span> <span class="n">continuous_cols</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span> <span class="o">=</span> <span class="n">quantization_setup</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">=</span> <span class="n">cols_to_scale</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span> <span class="o">=</span> <span class="n">auto_embed_dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span> <span class="o">=</span> <span class="n">embedding_rule</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">=</span> <span class="n">default_embed_dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">scale</span> <span class="o">=</span> <span class="n">scale</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="o">=</span> <span class="n">already_standard</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span> <span class="o">=</span> <span class="nb">bool</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">TabPreprocessor</span><span class="p">(</span>
                <span class="n">cat_embed_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">,</span>
                <span class="n">continuous_cols</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">,</span>
                <span class="n">quantization_setup</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">quantization_setup</span><span class="p">,</span>
                <span class="n">cols_to_scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span><span class="p">,</span>
                <span class="n">auto_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">auto_embed_dim</span><span class="p">,</span>
                <span class="n">embedding_rule</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">embedding_rule</span><span class="p">,</span>
                <span class="n">default_embed_dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
                <span class="n">verbose</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">,</span>
                <span class="n">scale</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">,</span>
                <span class="n">already_standard</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span><span class="p">,</span>
                <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">kwargs</span><span class="p">,</span>
            <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;DINPreprocessor&quot;</span><span class="p">:</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span> <span class="o">=</span> <span class="p">{</span>
                <span class="n">col</span><span class="p">:</span> <span class="n">i</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">col</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">keys</span><span class="p">())</span>
            <span class="p">}</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span> <span class="o">=</span> <span class="p">{}</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_fit_label_encoders</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]:</span>
        <span class="n">_df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pre_transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="n">df_w_sequences</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_build_sequences</span><span class="p">(</span><span class="n">_df</span><span class="p">)</span>
        <span class="n">X_all</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_concatenate_features</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">X_all</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">())</span>

    <span class="k">def</span> <span class="nf">fit_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">]:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">inverse_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">X</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span><span class="p">(</span>
            <span class="s2">&quot;inverse_transform is not implemented for this preprocessor&quot;</span>
        <span class="p">)</span>

    <span class="k">def</span> <span class="nf">_fit_label_encoders</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">item_le</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span><span class="n">columns_to_encode</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_item_col</span><span class="p">()])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">item_le</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">n_items</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">item_le</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">_get_item_col</span><span class="p">()]</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
        <span class="n">user_behaviour_embed_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embed_size</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_items</span>
        <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">user_behaviour_config</span> <span class="o">=</span> <span class="p">(</span>
            <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;item_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)],</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">n_items</span><span class="p">,</span>
            <span class="n">user_behaviour_embed_size</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_update_din_columns_idx</span><span class="p">(</span><span class="s2">&quot;item&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">[</span><span class="s2">&quot;target_item&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_fit_action_label_encoder</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_fit_other_seq_label_encoders</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_fit_action_label_encoder</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">action_le</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span><span class="n">columns_to_encode</span><span class="o">=</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">action_le</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">n_actions</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">action_le</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">])</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">action_seq_config</span> <span class="o">=</span> <span class="p">(</span>
            <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;action_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)],</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">n_actions</span><span class="p">,</span>
        <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_update_din_columns_idx</span><span class="p">(</span><span class="s2">&quot;action&quot;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_other_seq_cols_float_warning</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
        <span class="p">]</span>
        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">dtype</span> <span class="o">==</span> <span class="nb">float</span><span class="p">:</span>
                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
                    <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2"> is a float column. It will be converted to integers. &quot;</span>
                    <span class="s2">&quot;If this is not what you want, please convert it beforehand.&quot;</span><span class="p">,</span>
                    <span class="ne">UserWarning</span><span class="p">,</span>
                <span class="p">)</span>

    <span class="k">def</span> <span class="nf">_convert_other_seq_cols_to_int</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
        <span class="p">]</span>
        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">dtype</span> <span class="o">==</span> <span class="nb">float</span><span class="p">:</span>
                <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">astype</span><span class="p">(</span><span class="nb">int</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_fit_other_seq_label_encoders</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_other_seq_cols_float_warning</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_convert_other_seq_cols_to_int</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
            <span class="n">columns_to_encode</span><span class="o">=</span><span class="p">[</span>
                <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
                <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
            <span class="p">]</span>
        <span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
        <span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">n_other_seq_cols</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">col</span><span class="p">:</span> <span class="nb">max</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
        <span class="p">}</span>
        <span class="n">other_seq_embed_sizes</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">col</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_embed_size</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_other_seq_cols</span><span class="p">[</span><span class="n">col</span><span class="p">])</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
        <span class="p">}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_config</span> <span class="o">=</span> <span class="p">[</span>
            <span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_get_seq_col_names</span><span class="p">(</span><span class="n">col</span><span class="p">),</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">n_other_seq_cols</span><span class="p">[</span><span class="n">col</span><span class="p">],</span>
                <span class="n">other_seq_embed_sizes</span><span class="p">[</span><span class="n">col</span><span class="p">],</span>
            <span class="p">)</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
        <span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_update_din_columns_idx_for_other_seq</span><span class="p">(</span><span class="n">other_seq_cols</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_get_item_col</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="k">return</span> <span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span>
            <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">item_embed_col</span>
        <span class="p">)</span>

    <span class="k">def</span> <span class="nf">_get_embed_size</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]],</span> <span class="n">n_unique</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">int</span><span class="p">:</span>
        <span class="k">return</span> <span class="n">col</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">embed_sz_rule</span><span class="p">(</span><span class="n">n_unique</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_get_seq_col_names</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">col</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
        <span class="k">return</span> <span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)]</span>

    <span class="k">def</span> <span class="nf">_update_din_columns_idx</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">prefix</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">length</span><span class="p">:</span> <span class="nb">int</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">current_len</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
            <span class="p">{</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">prefix</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="n">current_len</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">length</span><span class="p">)}</span>
        <span class="p">)</span>

    <span class="k">def</span> <span class="nf">_update_din_columns_idx_for_other_seq</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">other_seq_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">current_len</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
                <span class="p">{</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="o">+</span><span class="mi">1</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="n">current_len</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">)}</span>
            <span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;target_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>
            <span class="n">current_len</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_pre_transform</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
        <span class="n">check_is_fitted</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">attributes</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;din_columns_idx&quot;</span><span class="p">])</span>
        <span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">item_le</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
            <span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_le</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
            <span class="n">df</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_le</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">df</span>

    <span class="k">def</span> <span class="nf">_build_sequences</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
        <span class="c1"># df.columns.tolist() -&gt; to avoid annoying pandas warning</span>
        <span class="k">return</span> <span class="p">(</span>
            <span class="n">df</span><span class="o">.</span><span class="n">groupby</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">)[</span><span class="n">df</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()]</span>  <span class="c1"># type: ignore[index]</span>
            <span class="o">.</span><span class="n">apply</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_group_sequences</span><span class="p">)</span>
            <span class="o">.</span><span class="n">reset_index</span><span class="p">(</span><span class="n">drop</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
        <span class="p">)</span>

    <span class="k">def</span> <span class="nf">_group_sequences</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">:</span>
        <span class="n">item_col</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_item_col</span><span class="p">()</span>
        <span class="n">items</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="n">item_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="n">targets</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="n">drop_cols</span> <span class="o">=</span> <span class="p">[</span><span class="n">item_col</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span>

        <span class="c1"># sequences cannot be built with user, item pairs with only one</span>
        <span class="c1"># interaction</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">items</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="mi">1</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">()</span>

        <span class="n">sequences</span> <span class="o">=</span> <span class="p">[</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_create_sequence</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">items</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">max</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">items</span><span class="p">)</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">))</span>
        <span class="p">]</span>

        <span class="n">seq_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">sequences</span><span class="p">)</span>
        <span class="n">non_seq_cols</span> <span class="o">=</span> <span class="n">group</span><span class="o">.</span><span class="n">drop_duplicates</span><span class="p">([</span><span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">])</span><span class="o">.</span><span class="n">drop</span><span class="p">(</span><span class="n">drop_cols</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">pd</span><span class="o">.</span><span class="n">merge</span><span class="p">(</span><span class="n">seq_df</span><span class="p">,</span> <span class="n">non_seq_cols</span><span class="p">,</span> <span class="n">on</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_create_sequence</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
        <span class="n">items</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
        <span class="n">targets</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
        <span class="n">i</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">drop_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]:</span>
        <span class="n">end_idx</span> <span class="o">=</span> <span class="nb">min</span><span class="p">(</span><span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">items</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
        <span class="n">item_sequences</span> <span class="o">=</span> <span class="n">items</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="n">end_idx</span><span class="p">]</span>
        <span class="n">target_item</span> <span class="o">=</span> <span class="n">items</span><span class="p">[</span><span class="n">end_idx</span><span class="p">]</span>
        <span class="n">target</span> <span class="o">=</span> <span class="n">targets</span><span class="p">[</span><span class="n">end_idx</span><span class="p">]</span>

        <span class="n">sequence</span> <span class="o">=</span> <span class="p">{</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">user_id_col</span><span class="p">:</span> <span class="n">group</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
            <span class="s2">&quot;items_sequence&quot;</span><span class="p">:</span> <span class="n">item_sequences</span><span class="p">,</span>
            <span class="s2">&quot;target_item&quot;</span><span class="p">:</span> <span class="n">target_item</span><span class="p">,</span>
        <span class="p">}</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
            <span class="n">sequence</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_create_action_sequence</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">targets</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">target</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">)</span>
            <span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">sequence</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">]</span> <span class="o">=</span> <span class="n">target</span>
            <span class="n">drop_cols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
            <span class="n">sequence</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_create_other_seq_sequence</span><span class="p">(</span><span class="n">group</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">))</span>

        <span class="k">return</span> <span class="n">sequence</span>

    <span class="k">def</span> <span class="nf">_create_action_sequence</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span>
        <span class="n">targets</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span>
        <span class="n">i</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">target</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">drop_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]:</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span> <span class="o">!=</span> <span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">:</span>
            <span class="n">actions</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
            <span class="n">action_sequences</span> <span class="o">=</span> <span class="p">(</span>
                <span class="n">actions</span><span class="p">[</span><span class="n">i</span> <span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">actions</span><span class="p">)</span>
                <span class="k">else</span> <span class="n">actions</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
            <span class="p">)</span>
            <span class="n">drop_cols</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">target</span> <span class="o">-=</span> <span class="mi">1</span>  <span class="c1"># the &#39;transform&#39; method adds 1 as it saves 0 for padding</span>
            <span class="n">action_sequences</span> <span class="o">=</span> <span class="p">(</span>
                <span class="n">targets</span><span class="p">[</span><span class="n">i</span> <span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">targets</span><span class="p">)</span>
                <span class="k">else</span> <span class="n">targets</span><span class="p">[</span><span class="n">i</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
            <span class="p">)</span>

        <span class="k">return</span> <span class="p">{</span><span class="bp">self</span><span class="o">.</span><span class="n">target_col</span><span class="p">:</span> <span class="n">target</span><span class="p">,</span> <span class="s2">&quot;actions_sequence&quot;</span><span class="p">:</span> <span class="n">action_sequences</span><span class="p">}</span>

    <span class="k">def</span> <span class="nf">_create_other_seq_sequence</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">group</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">i</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span> <span class="n">drop_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]:</span>
        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
        <span class="p">]</span>
        <span class="n">drop_cols</span> <span class="o">+=</span> <span class="n">other_seq_cols</span>
        <span class="n">other_seqs</span> <span class="o">=</span> <span class="p">{</span><span class="n">col</span><span class="p">:</span> <span class="n">group</span><span class="p">[</span><span class="n">col</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">}</span>

        <span class="n">other_seqs_sequences</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
            <span class="n">other_seqs_sequences</span><span class="p">[</span><span class="n">col</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
                <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="n">i</span> <span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">])</span>
                <span class="k">else</span> <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="n">i</span><span class="p">:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
            <span class="p">)</span>

        <span class="n">sequence</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">{}</span>
        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
            <span class="n">sequence</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_sequence&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">other_seqs_sequences</span><span class="p">[</span><span class="n">col</span><span class="p">]</span>
            <span class="n">sequence</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;target_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
                <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">]</span>
                <span class="k">if</span> <span class="n">i</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span> <span class="o">&lt;</span> <span class="nb">len</span><span class="p">(</span><span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">])</span>
                <span class="k">else</span> <span class="n">other_seqs</span><span class="p">[</span><span class="n">col</span><span class="p">][</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
            <span class="p">)</span>

        <span class="k">return</span> <span class="n">sequence</span>

    <span class="k">def</span> <span class="nf">_concatenate_features</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df_w_sequences</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
        <span class="n">user_behaviour_seq</span> <span class="o">=</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="s2">&quot;items_sequence&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
        <span class="n">X_user_behaviour</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span>
            <span class="p">[</span>
                <span class="n">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
                <span class="k">for</span> <span class="n">seq</span> <span class="ow">in</span> <span class="n">user_behaviour_seq</span>
            <span class="p">]</span>
        <span class="p">)</span>
        <span class="n">X_target_item</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">[</span><span class="s2">&quot;target_item&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">())</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
        <span class="n">X_all</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_user_behaviour</span><span class="p">,</span> <span class="n">X_target_item</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">has_standard_tab_data</span><span class="p">:</span>
            <span class="n">X_tab</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">transform</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">)</span>
            <span class="n">X_all</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_tab</span><span class="p">,</span> <span class="n">X_all</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">action_col</span><span class="p">:</span>
            <span class="n">action_seq</span> <span class="o">=</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="s2">&quot;actions_sequence&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>
            <span class="n">X_actions</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span>
                <span class="p">[</span>
                    <span class="n">pad_sequences</span><span class="p">(</span><span class="n">seq</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
                    <span class="k">for</span> <span class="n">seq</span> <span class="ow">in</span> <span class="n">action_seq</span>
                <span class="p">]</span>
            <span class="p">)</span>
            <span class="n">X_all</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_all</span><span class="p">,</span> <span class="n">X_actions</span><span class="p">],</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span><span class="p">:</span>
            <span class="n">X_all</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_concatenate_other_seq_features</span><span class="p">(</span><span class="n">df_w_sequences</span><span class="p">,</span> <span class="n">X_all</span><span class="p">)</span>

        <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span> <span class="o">==</span> <span class="n">X_all</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="p">(</span>
            <span class="sa">f</span><span class="s2">&quot;Something went wrong. The number of columns in the final array &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;(</span><span class="si">{</span><span class="n">X_all</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span><span class="si">}</span><span class="s2">) is different from the number of columns in &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;self.din_columns_idx (</span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">din_columns_idx</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
        <span class="p">)</span>

        <span class="k">return</span> <span class="n">X_all</span>

    <span class="k">def</span> <span class="nf">_concatenate_other_seq_features</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">df_w_sequences</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">X_all</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">np</span><span class="o">.</span><span class="n">ndarray</span><span class="p">:</span>
        <span class="n">other_seq_cols</span> <span class="o">=</span> <span class="p">[</span>
            <span class="n">col</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">col</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)</span> <span class="k">else</span> <span class="n">col</span>
            <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">other_seq_embed_cols</span>
        <span class="p">]</span>
        <span class="n">other_seq</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">col</span><span class="p">:</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">_sequence&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
        <span class="p">}</span>
        <span class="n">other_seq_target</span> <span class="o">=</span> <span class="p">{</span>
            <span class="n">col</span><span class="p">:</span> <span class="n">df_w_sequences</span><span class="p">[</span><span class="sa">f</span><span class="s2">&quot;target_</span><span class="si">{</span><span class="n">col</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">]</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span>
        <span class="p">}</span>
        <span class="n">X_other_seq_arrays</span> <span class="o">=</span> <span class="p">[]</span>

        <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">other_seq_cols</span><span class="p">:</span>
            <span class="n">X_other_seq_arrays</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                <span class="n">np</span><span class="o">.</span><span class="n">vstack</span><span class="p">(</span>
                    <span class="p">[</span>
                        <span class="n">pad_sequences</span><span class="p">(</span><span class="n">s</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_seq_length</span><span class="p">,</span> <span class="n">pad_idx</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
                        <span class="k">for</span> <span class="n">s</span> <span class="ow">in</span> <span class="n">other_seq</span><span class="p">[</span><span class="n">col</span><span class="p">]</span>
                    <span class="p">]</span>
                <span class="p">)</span>
            <span class="p">)</span>
            <span class="n">X_other_seq_arrays</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">(</span><span class="n">other_seq_target</span><span class="p">[</span><span class="n">col</span><span class="p">])</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">))</span>

        <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">concatenate</span><span class="p">([</span><span class="n">X_all</span><span class="p">]</span> <span class="o">+</span> <span class="n">X_other_seq_arrays</span><span class="p">,</span> <span class="n">axis</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">











  </div>

    </div>

</div><h2 id="chunked-versions">Chunked versions<a class="headerlink" href="#chunked-versions" title="Permanent link">&para;</a></h2>
<p>Chunked versions of the preprocessors are also available. These are useful
when the data is too big to fit in memory. See also the <a href="load_from_folder.html"><code>load_from_folder</code></a>
module in the library and the corresponding section here in the documentation.</p>
<p>Note that there is not a <code>ChunkImagePreprocessor</code>. This is because the
processing of the images will occur inside the <code>ImageFromFolder</code> class in
the <a href="load_from_folder.html"><code>load_from_folder</code></a> module.</p>


<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">ChunkWidePreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.WidePreprocessor">WidePreprocessor</a></code></p>


        <p>Preprocessor to prepare the wide input dataset</p>
<p>This Preprocessor prepares the data for the wide, linear component.
This linear model is implemented via an Embedding layer that is
connected to the output neuron. <code>ChunkWidePreprocessor</code> numerically
encodes all the unique values of all categorical columns <code>wide_cols +
crossed_cols</code>. See the Example below.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>wide_cols</code>
            </td>
            <td>
                  <code><span title="typing.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of strings with the name of the columns that will label
encoded and passed through the <code>wide</code> component</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>crossed_cols</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Tuple">Tuple</span>[str, str]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of Tuples with the name of the columns that will be <code>'crossed'</code>
and then label encoded. e.g. <em>[('education', 'occupation'), ...]</em>. For
binary features, a cross-product transformation is 1 if and only if
the constituent features are all 1, and 0 otherwise.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_crossed_cols">wide_crossed_cols</span></code></td>
            <td>
                  <code><span title="typing.List">List</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the names of all columns that will be label encoded</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.encoding_dict">encoding_dict</span></code></td>
            <td>
                  <code>Dict</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary where the keys are the result of pasting <code>colname + '_' +
column value</code> and the values are the corresponding mapped integer.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.inverse_encoding_dict">inverse_encoding_dict</span></code></td>
            <td>
                  <code>Dict</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>the inverse encoding dictionary</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.wide_dim">wide_dim</span></code></td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dimension of the wide model (i.e. dim of the linear layer)</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ChunkWidePreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">chunk</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;color&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;r&#39;</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="s1">&#39;g&#39;</span><span class="p">],</span> <span class="s1">&#39;size&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;s&#39;</span><span class="p">,</span> <span class="s1">&#39;n&#39;</span><span class="p">,</span> <span class="s1">&#39;l&#39;</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">wide_cols</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;color&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">crossed_cols</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;color&#39;</span><span class="p">,</span> <span class="s1">&#39;size&#39;</span><span class="p">)]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">chunk_wide_preprocessor</span> <span class="o">=</span> <span class="n">ChunkWidePreprocessor</span><span class="p">(</span><span class="n">wide_cols</span><span class="o">=</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="o">=</span><span class="n">crossed_cols</span><span class="p">,</span>
<span class="gp">... </span><span class="n">n_chunks</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_wide</span> <span class="o">=</span> <span class="n">chunk_wide_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">209</span>
<span class="normal">210</span>
<span class="normal">211</span>
<span class="normal">212</span>
<span class="normal">213</span>
<span class="normal">214</span>
<span class="normal">215</span>
<span class="normal">216</span>
<span class="normal">217</span>
<span class="normal">218</span>
<span class="normal">219</span>
<span class="normal">220</span>
<span class="normal">221</span>
<span class="normal">222</span>
<span class="normal">223</span>
<span class="normal">224</span>
<span class="normal">225</span>
<span class="normal">226</span>
<span class="normal">227</span>
<span class="normal">228</span>
<span class="normal">229</span>
<span class="normal">230</span>
<span class="normal">231</span>
<span class="normal">232</span>
<span class="normal">233</span>
<span class="normal">234</span>
<span class="normal">235</span>
<span class="normal">236</span>
<span class="normal">237</span>
<span class="normal">238</span>
<span class="normal">239</span>
<span class="normal">240</span>
<span class="normal">241</span>
<span class="normal">242</span>
<span class="normal">243</span>
<span class="normal">244</span>
<span class="normal">245</span>
<span class="normal">246</span>
<span class="normal">247</span>
<span class="normal">248</span>
<span class="normal">249</span>
<span class="normal">250</span>
<span class="normal">251</span>
<span class="normal">252</span>
<span class="normal">253</span>
<span class="normal">254</span>
<span class="normal">255</span>
<span class="normal">256</span>
<span class="normal">257</span>
<span class="normal">258</span>
<span class="normal">259</span>
<span class="normal">260</span>
<span class="normal">261</span>
<span class="normal">262</span>
<span class="normal">263</span>
<span class="normal">264</span>
<span class="normal">265</span>
<span class="normal">266</span>
<span class="normal">267</span>
<span class="normal">268</span>
<span class="normal">269</span>
<span class="normal">270</span>
<span class="normal">271</span>
<span class="normal">272</span>
<span class="normal">273</span>
<span class="normal">274</span>
<span class="normal">275</span>
<span class="normal">276</span>
<span class="normal">277</span>
<span class="normal">278</span>
<span class="normal">279</span>
<span class="normal">280</span>
<span class="normal">281</span>
<span class="normal">282</span>
<span class="normal">283</span>
<span class="normal">284</span>
<span class="normal">285</span>
<span class="normal">286</span>
<span class="normal">287</span>
<span class="normal">288</span>
<span class="normal">289</span>
<span class="normal">290</span>
<span class="normal">291</span>
<span class="normal">292</span>
<span class="normal">293</span>
<span class="normal">294</span>
<span class="normal">295</span>
<span class="normal">296</span>
<span class="normal">297</span>
<span class="normal">298</span>
<span class="normal">299</span>
<span class="normal">300</span>
<span class="normal">301</span>
<span class="normal">302</span>
<span class="normal">303</span>
<span class="normal">304</span>
<span class="normal">305</span>
<span class="normal">306</span>
<span class="normal">307</span>
<span class="normal">308</span>
<span class="normal">309</span>
<span class="normal">310</span>
<span class="normal">311</span>
<span class="normal">312</span>
<span class="normal">313</span>
<span class="normal">314</span>
<span class="normal">315</span>
<span class="normal">316</span>
<span class="normal">317</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkWidePreprocessor</span><span class="p">(</span><span class="n">WidePreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the wide input dataset</span>

<span class="sd">    This Preprocessor prepares the data for the wide, linear component.</span>
<span class="sd">    This linear model is implemented via an Embedding layer that is</span>
<span class="sd">    connected to the output neuron. `ChunkWidePreprocessor` numerically</span>
<span class="sd">    encodes all the unique values of all categorical columns `wide_cols +</span>
<span class="sd">    crossed_cols`. See the Example below.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    wide_cols: List</span>
<span class="sd">        List of strings with the name of the columns that will label</span>
<span class="sd">        encoded and passed through the `wide` component</span>
<span class="sd">    crossed_cols: List, default = None</span>
<span class="sd">        List of Tuples with the name of the columns that will be `&#39;crossed&#39;`</span>
<span class="sd">        and then label encoded. e.g. _[(&#39;education&#39;, &#39;occupation&#39;), ...]_. For</span>
<span class="sd">        binary features, a cross-product transformation is 1 if and only if</span>
<span class="sd">        the constituent features are all 1, and 0 otherwise.</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    wide_crossed_cols: List</span>
<span class="sd">        List with the names of all columns that will be label encoded</span>
<span class="sd">    encoding_dict: Dict</span>
<span class="sd">        Dictionary where the keys are the result of pasting `colname + &#39;_&#39; +</span>
<span class="sd">        column value` and the values are the corresponding mapped integer.</span>
<span class="sd">    inverse_encoding_dict: Dict</span>
<span class="sd">        the inverse encoding dictionary</span>
<span class="sd">    wide_dim: int</span>
<span class="sd">        Dimension of the wide model (i.e. dim of the linear layer)</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkWidePreprocessor</span>
<span class="sd">    &gt;&gt;&gt; chunk = pd.DataFrame({&#39;color&#39;: [&#39;r&#39;, &#39;b&#39;, &#39;g&#39;], &#39;size&#39;: [&#39;s&#39;, &#39;n&#39;, &#39;l&#39;]})</span>
<span class="sd">    &gt;&gt;&gt; wide_cols = [&#39;color&#39;]</span>
<span class="sd">    &gt;&gt;&gt; crossed_cols = [(&#39;color&#39;, &#39;size&#39;)]</span>
<span class="sd">    &gt;&gt;&gt; chunk_wide_preprocessor = ChunkWidePreprocessor(wide_cols=wide_cols, crossed_cols=crossed_cols,</span>
<span class="sd">    ... n_chunks=1)</span>
<span class="sd">    &gt;&gt;&gt; X_wide = chunk_wide_preprocessor.fit_transform(chunk)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">wide_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>
        <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">crossed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ChunkWidePreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">wide_cols</span><span class="p">,</span> <span class="n">crossed_cols</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">chunk</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkWidePreprocessor&quot;</span><span class="p">:</span>
<span class="w">        </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>

<span class="sd">        Parameters</span>
<span class="sd">        ----------</span>
<span class="sd">        chunk: pd.DataFrame</span>
<span class="sd">            Input pandas dataframe</span>

<span class="sd">        Returns</span>
<span class="sd">        -------</span>
<span class="sd">        ChunkWidePreprocessor</span>
<span class="sd">            `ChunkWidePreprocessor` fitted object</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span> <span class="o">=</span> <span class="n">df_wide</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span><span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">])</span>
            <span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span><span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">])</span>
            <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">+=</span> <span class="mi">1</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span><span class="p">)}</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">wide_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;unseen&quot;</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkWidePreprocessor&quot;</span><span class="p">:</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Runs `partial_fit`. This is just to override the fit method in the base</span>
<span class="sd">        class. This class is not designed or thought to run fit</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;wide_cols=</span><span class="si">{wide_cols}</span><span class="s2">&quot;</span><span class="p">]</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_chunks=</span><span class="si">{n_chunks}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">crossed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;crossed_cols=</span><span class="si">{crossed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;WidePreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">









<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">partial_fit</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.partial_fit" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">partial_fit</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Fits the Preprocessor and creates required attributes</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>chunk</code>
            </td>
            <td>
                  <code><span title="pandas.DataFrame">DataFrame</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Input pandas dataframe</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
      </tbody>
    </table>


    <p><span class="doc-section-title">Returns:</span></p>
    <table>
      <thead>
        <tr>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor" href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor">ChunkWidePreprocessor</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><code>ChunkWidePreprocessor</code> fitted object</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">267</span>
<span class="normal">268</span>
<span class="normal">269</span>
<span class="normal">270</span>
<span class="normal">271</span>
<span class="normal">272</span>
<span class="normal">273</span>
<span class="normal">274</span>
<span class="normal">275</span>
<span class="normal">276</span>
<span class="normal">277</span>
<span class="normal">278</span>
<span class="normal">279</span>
<span class="normal">280</span>
<span class="normal">281</span>
<span class="normal">282</span>
<span class="normal">283</span>
<span class="normal">284</span>
<span class="normal">285</span>
<span class="normal">286</span>
<span class="normal">287</span>
<span class="normal">288</span>
<span class="normal">289</span>
<span class="normal">290</span>
<span class="normal">291</span>
<span class="normal">292</span>
<span class="normal">293</span>
<span class="normal">294</span>
<span class="normal">295</span>
<span class="normal">296</span>
<span class="normal">297</span>
<span class="normal">298</span>
<span class="normal">299</span>
<span class="normal">300</span>
<span class="normal">301</span>
<span class="normal">302</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">chunk</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkWidePreprocessor&quot;</span><span class="p">:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Fits the Preprocessor and creates required attributes</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    chunk: pd.DataFrame</span>
<span class="sd">        Input pandas dataframe</span>

<span class="sd">    Returns</span>
<span class="sd">    -------</span>
<span class="sd">    ChunkWidePreprocessor</span>
<span class="sd">        `ChunkWidePreprocessor` fitted object</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">df_wide</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_wide</span><span class="p">(</span><span class="n">chunk</span><span class="p">)</span>
    <span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span> <span class="o">=</span> <span class="n">df_wide</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">()</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span> <span class="o">=</span> <span class="nb">set</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span><span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">])</span>
        <span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_make_global_feature_list</span><span class="p">(</span><span class="n">df_wide</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">wide_crossed_cols</span><span class="p">])</span>
        <span class="p">)</span>

    <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">+=</span> <span class="mi">1</span>

    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">:</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">v</span><span class="p">:</span> <span class="n">i</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">glob_feature_set</span><span class="p">)}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">wide_dim</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">()}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">inverse_encoding_dict</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;unseen&quot;</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

    <span class="k">return</span> <span class="bp">self</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>

<div class="doc doc-object doc-function">


<h3 id="pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit" class="doc doc-heading">
            <span class="doc doc-object-name doc-function-name">fit</span>


<a href="#pytorch_widedeep.preprocessing.wide_preprocessor.ChunkWidePreprocessor.fit" class="headerlink" title="Permanent link">&para;</a></h3>
<div class="doc-signature highlight"><pre><span></span><code><span class="nf">fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div>

    <div class="doc doc-contents ">

        <p>Runs <code>partial_fit</code>. This is just to override the fit method in the base
class. This class is not designed or thought to run fit</p>

            <details class="quote">
              <summary>Source code in <code>pytorch_widedeep/preprocessing/wide_preprocessor.py</code></summary>
              <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">304</span>
<span class="normal">305</span>
<span class="normal">306</span>
<span class="normal">307</span>
<span class="normal">308</span>
<span class="normal">309</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkWidePreprocessor&quot;</span><span class="p">:</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Runs `partial_fit`. This is just to override the fit method in the base</span>
<span class="sd">    class. This class is not designed or thought to run fit</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
            </details>
    </div>

</div>



  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">ChunkTabPreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor" href="#pytorch_widedeep.preprocessing.tab_preprocessor.TabPreprocessor">TabPreprocessor</a></code></p>


        <p>Preprocessor to prepare the <code>deeptabular</code> component input dataset</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>n_chunks</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Number of chunks that the tabular dataset is divided by.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>cat_embed_cols</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], <span title="pytorch_widedeep.wdtypes.List">List</span>[<span title="pytorch_widedeep.wdtypes.Tuple">Tuple</span>[str, int]]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List containing the name of the categorical columns that will be
represented by embeddings (e.g. <em>['education', 'relationship', ...]</em>) or
a Tuple with the name and the embedding dimension (e.g.: <em>[
('education',32), ('relationship',16), ...]</em>). <br/> Param alias:
<code>embed_cols</code></p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>continuous_cols</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the name of the continuous cols</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>cols_and_bins</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Continuous columns can be turned into categorical via
<code>pd.cut</code>. 'cols_and_bins' is dictionary where the keys are the column
names to quantize and the values are a list of scalars indicating the
bin edges. <br/> Param alias: <code>quantization_setup</code></p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>cols_to_scale</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Union">Union</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str], str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List with the names of the columns that will be standarised via
sklearn's <code>StandardScaler</code></p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>default_embed_dim</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dimension for the embeddings if the embed_dim is not provided in the
<code>cat_embed_cols</code> parameter and <code>auto_embed_dim</code> is set to
<code>False</code>.</p>
              </div>
            </td>
            <td>
                  <code>16</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>with_attention</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating whether the preprocessed data will be passed to an
attention-based model (more precisely a model where all embeddings
must have the same dimensions). If <code>True</code>, the param <code>cat_embed_cols</code>
must just be a list containing just the categorical column names:
e.g.
<em>['education', 'relationship', ...]</em>. This is because they will all be
 encoded using embeddings of the same dim, which will be specified
 later when the model is defined. <br/> Param alias:
 <code>for_transformer</code></p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>with_cls_token</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if a <code>'[CLS]'</code> token will be added to the dataset
when using attention-based models. The final hidden state
corresponding to this token is used as the aggregated representation
for classification and regression tasks. If not, the categorical
(and continuous embeddings if present) will be concatenated before
being passed to the final MLP (if present).</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>shared_embed</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the embeddings will be "shared" when using
attention-based models. The idea behind <code>shared_embed</code> is
described in the Appendix A in the <a href="https://arxiv.org/abs/2012.06678">TabTransformer paper</a>:
<em>'The goal of having column embedding is to enable the model to
distinguish the classes in one column from those in the other
columns'</em>. In other words, the idea is to let the model learn which
column is embedded at the time. See: <code>pytorch_widedeep.models.transformers._layers.SharedEmbeddings</code>.</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>verbose</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>scale</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
 releases. Please use <code>cols_to_scale</code> instead. <br/> Bool indicating
 whether or not to scale/standarise continuous cols. It is important
 to emphasize that all the DL models for tabular data in the library
 also include the possibility of normalising the input continuous
 features via a <code>BatchNorm</code> or a <code>LayerNorm</code>. <br/> Param alias:
 <code>scale_cont_cols</code>.</p>
              </div>
            </td>
            <td>
                  <code>False</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>already_standard</code>
            </td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.List">List</span>[str]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p><img alt="ℹ️" class="emojione" src="https://cdnjs.cloudflare.com/ajax/libs/emojione/2.2.7/assets/png/2139.png" title=":information_source:" /> <strong>note</strong>: this arg will be removed in upcoming
 releases. Please use <code>cols_to_scale</code> instead. <br/> List with the
 name of the continuous cols that do not need to be
 scaled/standarised.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Other Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code>**kwargs</code></td>
            <td>
            </td>
            <td>
              <div class="doc-md-description">
                <p><code>pd.cut</code> and <code>StandardScaler</code> related args</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cat_cols">cat_cols</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.List">List</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List containing the names of the categorical columns after processing.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cols_and_bins">cols_and_bins</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Optional">Optional</span>[<span title="pytorch_widedeep.wdtypes.Dict">Dict</span>[str, <span title="pytorch_widedeep.wdtypes.Union">Union</span>[int, <span title="pytorch_widedeep.wdtypes.List">List</span>[float]]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary containing the quantization setup after processing if
quantization is requested. This is derived from the
quantization_setup parameter.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.quant_args">quant_args</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary containing arguments passed to pandas.cut() function for quantization.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.scale_args">scale_args</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary containing arguments passed to StandardScaler.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.label_encoder">label_encoder</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.deeptabular_utils.LabelEncoder" href="utils/deeptabular_utils.html#pytorch_widedeep.utils.deeptabular_utils.LabelEncoder">LabelEncoder</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Instance of LabelEncoder used to encode categorical variables.
See <code>pytorch_widedeep.utils.dense_utils.LabelEncoder</code>.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.cat_embed_input">cat_embed_input</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of tuples with the column name, number of individual values for
that column and, if <code>with_attention</code> is set to <code>False</code>, the
corresponding embeddings dim, e.g. <em>[('education', 16, 10),
('relationship', 6, 8), ...]</em>.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.standardize_cols">standardize_cols</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.List">List</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>List of the columns that will be standardized.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.scaler">scaler</span></code></td>
            <td>
                  <code><span title="sklearn.preprocessing.StandardScaler">StandardScaler</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>An instance of <code>sklearn.preprocessing.StandardScaler</code> if standardization
is requested.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.column_idx">column_idx</span></code></td>
            <td>
                  <code><span title="pytorch_widedeep.wdtypes.Dict">Dict</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Dictionary where keys are column names and values are column indexes.
This is necessary to slice tensors.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.quantizer">quantizer</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer" href="#pytorch_widedeep.preprocessing.tab_preprocessor.Quantizer">Quantizer</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>An instance of <code>Quantizer</code> if quantization is requested.</p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.tab_preprocessor.ChunkTabPreprocessor.is_fitted">is_fitted</span></code></td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the preprocessor has been fitted.</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ChunkTabPreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">42</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">chunk_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;cat_col&#39;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">choice</span><span class="p">([</span><span class="s1">&#39;A&#39;</span><span class="p">,</span> <span class="s1">&#39;B&#39;</span><span class="p">,</span> <span class="s1">&#39;C&#39;</span><span class="p">],</span> <span class="n">size</span><span class="o">=</span><span class="mi">8</span><span class="p">),</span>
<span class="gp">... </span><span class="s1">&#39;cont_col&#39;</span><span class="p">:</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">uniform</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">100</span><span class="p">,</span> <span class="n">size</span><span class="o">=</span><span class="mi">8</span><span class="p">)})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cat_embed_cols</span> <span class="o">=</span> <span class="p">[(</span><span class="s1">&#39;cat_col&#39;</span><span class="p">,</span><span class="mi">4</span><span class="p">)]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cont_cols</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;cont_col&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tab_preprocessor</span> <span class="o">=</span> <span class="n">ChunkTabPreprocessor</span><span class="p">(</span>
<span class="gp">... </span><span class="n">n_chunks</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cat_embed_cols</span><span class="o">=</span><span class="n">cat_embed_cols</span><span class="p">,</span> <span class="n">continuous_cols</span><span class="o">=</span><span class="n">cont_cols</span>
<span class="gp">... </span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_tab</span> <span class="o">=</span> <span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk_df</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">cat_embed_cols</span>
<span class="go">[(&#39;cat_col&#39;, 4)]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tab_preprocessor</span><span class="o">.</span><span class="n">column_idx</span>
<span class="go">{&#39;cat_col&#39;: 0, &#39;cont_col&#39;: 1}</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/tab_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"> 714</span>
<span class="normal"> 715</span>
<span class="normal"> 716</span>
<span class="normal"> 717</span>
<span class="normal"> 718</span>
<span class="normal"> 719</span>
<span class="normal"> 720</span>
<span class="normal"> 721</span>
<span class="normal"> 722</span>
<span class="normal"> 723</span>
<span class="normal"> 724</span>
<span class="normal"> 725</span>
<span class="normal"> 726</span>
<span class="normal"> 727</span>
<span class="normal"> 728</span>
<span class="normal"> 729</span>
<span class="normal"> 730</span>
<span class="normal"> 731</span>
<span class="normal"> 732</span>
<span class="normal"> 733</span>
<span class="normal"> 734</span>
<span class="normal"> 735</span>
<span class="normal"> 736</span>
<span class="normal"> 737</span>
<span class="normal"> 738</span>
<span class="normal"> 739</span>
<span class="normal"> 740</span>
<span class="normal"> 741</span>
<span class="normal"> 742</span>
<span class="normal"> 743</span>
<span class="normal"> 744</span>
<span class="normal"> 745</span>
<span class="normal"> 746</span>
<span class="normal"> 747</span>
<span class="normal"> 748</span>
<span class="normal"> 749</span>
<span class="normal"> 750</span>
<span class="normal"> 751</span>
<span class="normal"> 752</span>
<span class="normal"> 753</span>
<span class="normal"> 754</span>
<span class="normal"> 755</span>
<span class="normal"> 756</span>
<span class="normal"> 757</span>
<span class="normal"> 758</span>
<span class="normal"> 759</span>
<span class="normal"> 760</span>
<span class="normal"> 761</span>
<span class="normal"> 762</span>
<span class="normal"> 763</span>
<span class="normal"> 764</span>
<span class="normal"> 765</span>
<span class="normal"> 766</span>
<span class="normal"> 767</span>
<span class="normal"> 768</span>
<span class="normal"> 769</span>
<span class="normal"> 770</span>
<span class="normal"> 771</span>
<span class="normal"> 772</span>
<span class="normal"> 773</span>
<span class="normal"> 774</span>
<span class="normal"> 775</span>
<span class="normal"> 776</span>
<span class="normal"> 777</span>
<span class="normal"> 778</span>
<span class="normal"> 779</span>
<span class="normal"> 780</span>
<span class="normal"> 781</span>
<span class="normal"> 782</span>
<span class="normal"> 783</span>
<span class="normal"> 784</span>
<span class="normal"> 785</span>
<span class="normal"> 786</span>
<span class="normal"> 787</span>
<span class="normal"> 788</span>
<span class="normal"> 789</span>
<span class="normal"> 790</span>
<span class="normal"> 791</span>
<span class="normal"> 792</span>
<span class="normal"> 793</span>
<span class="normal"> 794</span>
<span class="normal"> 795</span>
<span class="normal"> 796</span>
<span class="normal"> 797</span>
<span class="normal"> 798</span>
<span class="normal"> 799</span>
<span class="normal"> 800</span>
<span class="normal"> 801</span>
<span class="normal"> 802</span>
<span class="normal"> 803</span>
<span class="normal"> 804</span>
<span class="normal"> 805</span>
<span class="normal"> 806</span>
<span class="normal"> 807</span>
<span class="normal"> 808</span>
<span class="normal"> 809</span>
<span class="normal"> 810</span>
<span class="normal"> 811</span>
<span class="normal"> 812</span>
<span class="normal"> 813</span>
<span class="normal"> 814</span>
<span class="normal"> 815</span>
<span class="normal"> 816</span>
<span class="normal"> 817</span>
<span class="normal"> 818</span>
<span class="normal"> 819</span>
<span class="normal"> 820</span>
<span class="normal"> 821</span>
<span class="normal"> 822</span>
<span class="normal"> 823</span>
<span class="normal"> 824</span>
<span class="normal"> 825</span>
<span class="normal"> 826</span>
<span class="normal"> 827</span>
<span class="normal"> 828</span>
<span class="normal"> 829</span>
<span class="normal"> 830</span>
<span class="normal"> 831</span>
<span class="normal"> 832</span>
<span class="normal"> 833</span>
<span class="normal"> 834</span>
<span class="normal"> 835</span>
<span class="normal"> 836</span>
<span class="normal"> 837</span>
<span class="normal"> 838</span>
<span class="normal"> 839</span>
<span class="normal"> 840</span>
<span class="normal"> 841</span>
<span class="normal"> 842</span>
<span class="normal"> 843</span>
<span class="normal"> 844</span>
<span class="normal"> 845</span>
<span class="normal"> 846</span>
<span class="normal"> 847</span>
<span class="normal"> 848</span>
<span class="normal"> 849</span>
<span class="normal"> 850</span>
<span class="normal"> 851</span>
<span class="normal"> 852</span>
<span class="normal"> 853</span>
<span class="normal"> 854</span>
<span class="normal"> 855</span>
<span class="normal"> 856</span>
<span class="normal"> 857</span>
<span class="normal"> 858</span>
<span class="normal"> 859</span>
<span class="normal"> 860</span>
<span class="normal"> 861</span>
<span class="normal"> 862</span>
<span class="normal"> 863</span>
<span class="normal"> 864</span>
<span class="normal"> 865</span>
<span class="normal"> 866</span>
<span class="normal"> 867</span>
<span class="normal"> 868</span>
<span class="normal"> 869</span>
<span class="normal"> 870</span>
<span class="normal"> 871</span>
<span class="normal"> 872</span>
<span class="normal"> 873</span>
<span class="normal"> 874</span>
<span class="normal"> 875</span>
<span class="normal"> 876</span>
<span class="normal"> 877</span>
<span class="normal"> 878</span>
<span class="normal"> 879</span>
<span class="normal"> 880</span>
<span class="normal"> 881</span>
<span class="normal"> 882</span>
<span class="normal"> 883</span>
<span class="normal"> 884</span>
<span class="normal"> 885</span>
<span class="normal"> 886</span>
<span class="normal"> 887</span>
<span class="normal"> 888</span>
<span class="normal"> 889</span>
<span class="normal"> 890</span>
<span class="normal"> 891</span>
<span class="normal"> 892</span>
<span class="normal"> 893</span>
<span class="normal"> 894</span>
<span class="normal"> 895</span>
<span class="normal"> 896</span>
<span class="normal"> 897</span>
<span class="normal"> 898</span>
<span class="normal"> 899</span>
<span class="normal"> 900</span>
<span class="normal"> 901</span>
<span class="normal"> 902</span>
<span class="normal"> 903</span>
<span class="normal"> 904</span>
<span class="normal"> 905</span>
<span class="normal"> 906</span>
<span class="normal"> 907</span>
<span class="normal"> 908</span>
<span class="normal"> 909</span>
<span class="normal"> 910</span>
<span class="normal"> 911</span>
<span class="normal"> 912</span>
<span class="normal"> 913</span>
<span class="normal"> 914</span>
<span class="normal"> 915</span>
<span class="normal"> 916</span>
<span class="normal"> 917</span>
<span class="normal"> 918</span>
<span class="normal"> 919</span>
<span class="normal"> 920</span>
<span class="normal"> 921</span>
<span class="normal"> 922</span>
<span class="normal"> 923</span>
<span class="normal"> 924</span>
<span class="normal"> 925</span>
<span class="normal"> 926</span>
<span class="normal"> 927</span>
<span class="normal"> 928</span>
<span class="normal"> 929</span>
<span class="normal"> 930</span>
<span class="normal"> 931</span>
<span class="normal"> 932</span>
<span class="normal"> 933</span>
<span class="normal"> 934</span>
<span class="normal"> 935</span>
<span class="normal"> 936</span>
<span class="normal"> 937</span>
<span class="normal"> 938</span>
<span class="normal"> 939</span>
<span class="normal"> 940</span>
<span class="normal"> 941</span>
<span class="normal"> 942</span>
<span class="normal"> 943</span>
<span class="normal"> 944</span>
<span class="normal"> 945</span>
<span class="normal"> 946</span>
<span class="normal"> 947</span>
<span class="normal"> 948</span>
<span class="normal"> 949</span>
<span class="normal"> 950</span>
<span class="normal"> 951</span>
<span class="normal"> 952</span>
<span class="normal"> 953</span>
<span class="normal"> 954</span>
<span class="normal"> 955</span>
<span class="normal"> 956</span>
<span class="normal"> 957</span>
<span class="normal"> 958</span>
<span class="normal"> 959</span>
<span class="normal"> 960</span>
<span class="normal"> 961</span>
<span class="normal"> 962</span>
<span class="normal"> 963</span>
<span class="normal"> 964</span>
<span class="normal"> 965</span>
<span class="normal"> 966</span>
<span class="normal"> 967</span>
<span class="normal"> 968</span>
<span class="normal"> 969</span>
<span class="normal"> 970</span>
<span class="normal"> 971</span>
<span class="normal"> 972</span>
<span class="normal"> 973</span>
<span class="normal"> 974</span>
<span class="normal"> 975</span>
<span class="normal"> 976</span>
<span class="normal"> 977</span>
<span class="normal"> 978</span>
<span class="normal"> 979</span>
<span class="normal"> 980</span>
<span class="normal"> 981</span>
<span class="normal"> 982</span>
<span class="normal"> 983</span>
<span class="normal"> 984</span>
<span class="normal"> 985</span>
<span class="normal"> 986</span>
<span class="normal"> 987</span>
<span class="normal"> 988</span>
<span class="normal"> 989</span>
<span class="normal"> 990</span>
<span class="normal"> 991</span>
<span class="normal"> 992</span>
<span class="normal"> 993</span>
<span class="normal"> 994</span>
<span class="normal"> 995</span>
<span class="normal"> 996</span>
<span class="normal"> 997</span>
<span class="normal"> 998</span>
<span class="normal"> 999</span>
<span class="normal">1000</span>
<span class="normal">1001</span>
<span class="normal">1002</span>
<span class="normal">1003</span>
<span class="normal">1004</span>
<span class="normal">1005</span>
<span class="normal">1006</span>
<span class="normal">1007</span>
<span class="normal">1008</span>
<span class="normal">1009</span>
<span class="normal">1010</span>
<span class="normal">1011</span>
<span class="normal">1012</span>
<span class="normal">1013</span>
<span class="normal">1014</span>
<span class="normal">1015</span>
<span class="normal">1016</span>
<span class="normal">1017</span>
<span class="normal">1018</span>
<span class="normal">1019</span>
<span class="normal">1020</span>
<span class="normal">1021</span>
<span class="normal">1022</span>
<span class="normal">1023</span>
<span class="normal">1024</span>
<span class="normal">1025</span>
<span class="normal">1026</span>
<span class="normal">1027</span>
<span class="normal">1028</span>
<span class="normal">1029</span>
<span class="normal">1030</span>
<span class="normal">1031</span>
<span class="normal">1032</span>
<span class="normal">1033</span>
<span class="normal">1034</span>
<span class="normal">1035</span>
<span class="normal">1036</span>
<span class="normal">1037</span>
<span class="normal">1038</span>
<span class="normal">1039</span>
<span class="normal">1040</span>
<span class="normal">1041</span>
<span class="normal">1042</span>
<span class="normal">1043</span>
<span class="normal">1044</span>
<span class="normal">1045</span>
<span class="normal">1046</span>
<span class="normal">1047</span>
<span class="normal">1048</span>
<span class="normal">1049</span>
<span class="normal">1050</span>
<span class="normal">1051</span>
<span class="normal">1052</span>
<span class="normal">1053</span>
<span class="normal">1054</span>
<span class="normal">1055</span>
<span class="normal">1056</span>
<span class="normal">1057</span>
<span class="normal">1058</span>
<span class="normal">1059</span>
<span class="normal">1060</span>
<span class="normal">1061</span>
<span class="normal">1062</span>
<span class="normal">1063</span>
<span class="normal">1064</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkTabPreprocessor</span><span class="p">(</span><span class="n">TabPreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the `deeptabular` component input dataset</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    n_chunks: int</span>
<span class="sd">        Number of chunks that the tabular dataset is divided by.</span>
<span class="sd">    cat_embed_cols: List, default = None</span>
<span class="sd">        List containing the name of the categorical columns that will be</span>
<span class="sd">        represented by embeddings (e.g. _[&#39;education&#39;, &#39;relationship&#39;, ...]_) or</span>
<span class="sd">        a Tuple with the name and the embedding dimension (e.g.: _[</span>
<span class="sd">        (&#39;education&#39;,32), (&#39;relationship&#39;,16), ...]_). &lt;br/&gt; Param alias:</span>
<span class="sd">        `embed_cols`</span>
<span class="sd">    continuous_cols: List, default = None</span>
<span class="sd">        List with the name of the continuous cols</span>
<span class="sd">    cols_and_bins: Dict, default = None</span>
<span class="sd">        Continuous columns can be turned into categorical via</span>
<span class="sd">        `pd.cut`. &#39;cols_and_bins&#39; is dictionary where the keys are the column</span>
<span class="sd">        names to quantize and the values are a list of scalars indicating the</span>
<span class="sd">        bin edges. &lt;br/&gt; Param alias: `quantization_setup`</span>
<span class="sd">    cols_to_scale: List, default = None,</span>
<span class="sd">        List with the names of the columns that will be standarised via</span>
<span class="sd">        sklearn&#39;s `StandardScaler`</span>
<span class="sd">    default_embed_dim: int, default=16</span>
<span class="sd">        Dimension for the embeddings if the embed_dim is not provided in the</span>
<span class="sd">        `cat_embed_cols` parameter and `auto_embed_dim` is set to</span>
<span class="sd">        `False`.</span>
<span class="sd">    with_attention: bool, default = False</span>
<span class="sd">        Boolean indicating whether the preprocessed data will be passed to an</span>
<span class="sd">        attention-based model (more precisely a model where all embeddings</span>
<span class="sd">        must have the same dimensions). If `True`, the param `cat_embed_cols`</span>
<span class="sd">        must just be a list containing just the categorical column names:</span>
<span class="sd">        e.g.</span>
<span class="sd">        _[&#39;education&#39;, &#39;relationship&#39;, ...]_. This is because they will all be</span>
<span class="sd">         encoded using embeddings of the same dim, which will be specified</span>
<span class="sd">         later when the model is defined. &lt;br/&gt; Param alias:</span>
<span class="sd">         `for_transformer`</span>
<span class="sd">    with_cls_token: bool, default = False</span>
<span class="sd">        Boolean indicating if a `&#39;[CLS]&#39;` token will be added to the dataset</span>
<span class="sd">        when using attention-based models. The final hidden state</span>
<span class="sd">        corresponding to this token is used as the aggregated representation</span>
<span class="sd">        for classification and regression tasks. If not, the categorical</span>
<span class="sd">        (and continuous embeddings if present) will be concatenated before</span>
<span class="sd">        being passed to the final MLP (if present).</span>
<span class="sd">    shared_embed: bool, default = False</span>
<span class="sd">        Boolean indicating if the embeddings will be &quot;shared&quot; when using</span>
<span class="sd">        attention-based models. The idea behind `shared_embed` is</span>
<span class="sd">        described in the Appendix A in the [TabTransformer paper](https://arxiv.org/abs/2012.06678):</span>
<span class="sd">        _&#39;The goal of having column embedding is to enable the model to</span>
<span class="sd">        distinguish the classes in one column from those in the other</span>
<span class="sd">        columns&#39;_. In other words, the idea is to let the model learn which</span>
<span class="sd">        column is embedded at the time. See: `pytorch_widedeep.models.transformers._layers.SharedEmbeddings`.</span>
<span class="sd">    verbose: int, default = 1</span>
<span class="sd">    scale: bool, default = False</span>
<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; Bool indicating</span>
<span class="sd">         whether or not to scale/standarise continuous cols. It is important</span>
<span class="sd">         to emphasize that all the DL models for tabular data in the library</span>
<span class="sd">         also include the possibility of normalising the input continuous</span>
<span class="sd">         features via a `BatchNorm` or a `LayerNorm`. &lt;br/&gt; Param alias:</span>
<span class="sd">         `scale_cont_cols`.</span>
<span class="sd">    already_standard: List, default = None</span>
<span class="sd">        :information_source: **note**: this arg will be removed in upcoming</span>
<span class="sd">         releases. Please use `cols_to_scale` instead. &lt;br/&gt; List with the</span>
<span class="sd">         name of the continuous cols that do not need to be</span>
<span class="sd">         scaled/standarised.</span>

<span class="sd">    Other Parameters</span>
<span class="sd">    ----------------</span>
<span class="sd">    **kwargs: dict</span>
<span class="sd">        `pd.cut` and `StandardScaler` related args</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    cat_cols: List[str]</span>
<span class="sd">        List containing the names of the categorical columns after processing.</span>
<span class="sd">    cols_and_bins: Optional[Dict[str, Union[int, List[float]]]]</span>
<span class="sd">        Dictionary containing the quantization setup after processing if</span>
<span class="sd">        quantization is requested. This is derived from the</span>
<span class="sd">        quantization_setup parameter.</span>
<span class="sd">    quant_args: Dict</span>
<span class="sd">        Dictionary containing arguments passed to pandas.cut() function for quantization.</span>
<span class="sd">    scale_args: Dict</span>
<span class="sd">        Dictionary containing arguments passed to StandardScaler.</span>
<span class="sd">    label_encoder: LabelEncoder</span>
<span class="sd">        Instance of LabelEncoder used to encode categorical variables.</span>
<span class="sd">        See `pytorch_widedeep.utils.dense_utils.LabelEncoder`.</span>
<span class="sd">    cat_embed_input: List</span>
<span class="sd">        List of tuples with the column name, number of individual values for</span>
<span class="sd">        that column and, if `with_attention` is set to `False`, the</span>
<span class="sd">        corresponding embeddings dim, e.g. _[(&#39;education&#39;, 16, 10),</span>
<span class="sd">        (&#39;relationship&#39;, 6, 8), ...]_.</span>
<span class="sd">    standardize_cols: List</span>
<span class="sd">        List of the columns that will be standardized.</span>
<span class="sd">    scaler: StandardScaler</span>
<span class="sd">        An instance of `sklearn.preprocessing.StandardScaler` if standardization</span>
<span class="sd">        is requested.</span>
<span class="sd">    column_idx: Dict</span>
<span class="sd">        Dictionary where keys are column names and values are column indexes.</span>
<span class="sd">        This is necessary to slice tensors.</span>
<span class="sd">    quantizer: Quantizer</span>
<span class="sd">        An instance of `Quantizer` if quantization is requested.</span>
<span class="sd">    is_fitted: bool</span>
<span class="sd">        Boolean indicating if the preprocessor has been fitted.</span>

<span class="sd">    Examples</span>
<span class="sd">    --------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; import numpy as np</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTabPreprocessor</span>
<span class="sd">    &gt;&gt;&gt; np.random.seed(42)</span>
<span class="sd">    &gt;&gt;&gt; chunk_df = pd.DataFrame({&#39;cat_col&#39;: np.random.choice([&#39;A&#39;, &#39;B&#39;, &#39;C&#39;], size=8),</span>
<span class="sd">    ... &#39;cont_col&#39;: np.random.uniform(1, 100, size=8)})</span>
<span class="sd">    &gt;&gt;&gt; cat_embed_cols = [(&#39;cat_col&#39;,4)]</span>
<span class="sd">    &gt;&gt;&gt; cont_cols = [&#39;cont_col&#39;]</span>
<span class="sd">    &gt;&gt;&gt; tab_preprocessor = ChunkTabPreprocessor(</span>
<span class="sd">    ... n_chunks=1, cat_embed_cols=cat_embed_cols, continuous_cols=cont_cols</span>
<span class="sd">    ... )</span>
<span class="sd">    &gt;&gt;&gt; X_tab = tab_preprocessor.fit_transform(chunk_df)</span>
<span class="sd">    &gt;&gt;&gt; tab_preprocessor.cat_embed_cols</span>
<span class="sd">    [(&#39;cat_col&#39;, 4)]</span>
<span class="sd">    &gt;&gt;&gt; tab_preprocessor.column_idx</span>
<span class="sd">    {&#39;cat_col&#39;: 0, &#39;cont_col&#39;: 1}</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;with_attention&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;for_transformer&quot;</span><span class="p">])</span>
    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;embed_cols&quot;</span><span class="p">])</span>
    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;scale&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;scale_cont_cols&quot;</span><span class="p">])</span>
    <span class="nd">@alias</span><span class="p">(</span><span class="s2">&quot;cols_and_bins&quot;</span><span class="p">,</span> <span class="p">[</span><span class="s2">&quot;quantization_setup&quot;</span><span class="p">])</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">cat_embed_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">continuous_cols</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">cols_and_bins</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">float</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">cols_to_scale</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">default_embed_dim</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">16</span><span class="p">,</span>
        <span class="n">with_attention</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">with_cls_token</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">shared_embed</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
        <span class="o">*</span><span class="p">,</span>
        <span class="n">scale</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">already_standard</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ChunkTabPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
            <span class="n">cat_embed_cols</span><span class="o">=</span><span class="n">cat_embed_cols</span><span class="p">,</span>
            <span class="n">continuous_cols</span><span class="o">=</span><span class="n">continuous_cols</span><span class="p">,</span>
            <span class="n">quantization_setup</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
            <span class="n">cols_to_scale</span><span class="o">=</span><span class="n">cols_to_scale</span><span class="p">,</span>
            <span class="n">auto_embed_dim</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
            <span class="n">embedding_rule</span><span class="o">=</span><span class="s2">&quot;google&quot;</span><span class="p">,</span>  <span class="c1"># does not matter, irrelevant</span>
            <span class="n">default_embed_dim</span><span class="o">=</span><span class="n">default_embed_dim</span><span class="p">,</span>
            <span class="n">with_attention</span><span class="o">=</span><span class="n">with_attention</span><span class="p">,</span>
            <span class="n">with_cls_token</span><span class="o">=</span><span class="n">with_cls_token</span><span class="p">,</span>
            <span class="n">shared_embed</span><span class="o">=</span><span class="n">shared_embed</span><span class="p">,</span>
            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
            <span class="n">scale</span><span class="o">=</span><span class="n">scale</span><span class="p">,</span>
            <span class="n">already_standard</span><span class="o">=</span><span class="n">already_standard</span><span class="p">,</span>
            <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="o">=</span> <span class="n">cols_and_bins</span>  <span class="c1"># type: ignore[assignment]</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">quantizer</span> <span class="o">=</span> <span class="n">Quantizer</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="p">,</span> <span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTabPreprocessor&quot;</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
        <span class="c1"># df here, and throughout the class, is a chunk of the original df</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">+=</span> <span class="mi">1</span>

        <span class="n">df_adj</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_insert_cls_token</span><span class="p">(</span><span class="n">df</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="p">{}</span>

        <span class="c1"># Categorical embeddings logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span><span class="p">:</span>
                <span class="n">df_cat</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_categorical</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span> <span class="o">=</span> <span class="n">LabelEncoder</span><span class="p">(</span>
                    <span class="n">columns_to_encode</span><span class="o">=</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="o">.</span><span class="n">tolist</span><span class="p">(),</span>
                    <span class="n">shared_embed</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">,</span>
                    <span class="n">with_attention</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">,</span>
                <span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">df_cat</span> <span class="o">=</span> <span class="n">df_adj</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">]</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df_cat</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">({</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cat</span><span class="o">.</span><span class="n">columns</span><span class="p">)})</span>

        <span class="c1"># Continuous columns logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span><span class="p">:</span>
                <span class="n">df_cont</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_prepare_continuous</span><span class="p">(</span><span class="n">df_adj</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">df_cont</span> <span class="o">=</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">]</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="o">.</span><span class="n">update</span><span class="p">(</span>
                <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">v</span> <span class="o">+</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">column_idx</span><span class="p">)</span> <span class="k">for</span> <span class="n">v</span><span class="p">,</span> <span class="n">k</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">df_cont</span><span class="o">.</span><span class="n">columns</span><span class="p">)}</span>
            <span class="p">)</span>

            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df_cont</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span><span class="p">]</span><span class="o">.</span><span class="n">values</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span>
                    <span class="n">Union</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">],</span> <span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]</span>
                <span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>

            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">label_encoder</span><span class="o">.</span><span class="n">encoding_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">)))</span>
                    <span class="k">else</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">k</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">v</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_dim</span><span class="p">[</span><span class="n">k</span><span class="p">]))</span>

            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>  <span class="c1"># just to make mypy happy</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
                    <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">:</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">((</span><span class="n">col</span><span class="p">,</span> <span class="n">n_cat</span><span class="p">))</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_input</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cont_embed_dim</span><span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTabPreprocessor&quot;</span><span class="p">:</span>
        <span class="c1"># just to override the fit method in the base class. This class is not</span>
        <span class="c1"># designed or thought to run fit</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_prepare_categorical</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]:</span>
        <span class="c1"># When dealing with chunks we will NOT support the option of</span>
        <span class="c1"># automatically define embeddings as this implies going through the</span>
        <span class="c1"># entire dataset</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="nb">tuple</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="n">emb</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">emb</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">]</span>
            <span class="n">cat_embed_dim</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span><span class="p">)</span>  <span class="c1"># type: ignore</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[assignment]</span>
            <span class="n">cat_embed_dim</span> <span class="o">=</span> <span class="p">{</span>
                <span class="n">e</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="k">for</span> <span class="n">e</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span>  <span class="c1"># type: ignore[misc]</span>
            <span class="p">}</span>  <span class="c1"># type: ignore</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">embed_prepared</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">cat_cols</span><span class="p">],</span> <span class="n">cat_embed_dim</span>

    <span class="k">def</span> <span class="nf">_prepare_continuous</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span>
    <span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">,</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]]:</span>
        <span class="c1"># Standardization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="o">!=</span> <span class="s2">&quot;all&quot;</span>
                <span class="k">else</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
            <span class="p">)</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="p">[</span>
                    <span class="n">c</span>
                    <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>  <span class="c1"># type: ignore[misc]</span>
                    <span class="k">if</span> <span class="n">c</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span>
                <span class="p">]</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">standardize_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">scaler</span> <span class="o">=</span> <span class="n">StandardScaler</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span>
        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Continuous columns will not be normalised&quot;</span><span class="p">)</span>

        <span class="c1"># Quantization logic</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="c1"># the quantized columns are then treated as categorical</span>
            <span class="n">quant_cont_embed_input</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="nb">int</span><span class="p">,</span> <span class="nb">int</span><span class="p">]]]</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">col</span><span class="p">,</span> <span class="n">val</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">val</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
                    <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                        <span class="p">(</span>
                            <span class="n">col</span><span class="p">,</span>
                            <span class="n">val</span><span class="p">,</span>
                            <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
                        <span class="p">)</span>
                    <span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">quant_cont_embed_input</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                        <span class="p">(</span>
                            <span class="n">col</span><span class="p">,</span>
                            <span class="nb">len</span><span class="p">(</span><span class="n">val</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span><span class="p">,</span>
                            <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span><span class="p">,</span>
                        <span class="p">)</span>
                    <span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">quant_cont_embed_input</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">continuous_prepared</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="n">df</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span><span class="p">],</span> <span class="n">quant_cont_embed_input</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>  <span class="c1"># noqa: C901</span>
        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_chunks=</span><span class="si">{n_chunks}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cat_embed_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cat_embed_cols=</span><span class="si">{cat_embed_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">continuous_cols</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;continuous_cols=</span><span class="si">{continuous_cols}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_and_bins</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cols_and_bins=</span><span class="si">{cols_and_bins}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cols_to_scale</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;cols_to_scale=</span><span class="si">{cols_to_scale}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">default_embed_dim</span> <span class="o">!=</span> <span class="mi">16</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;default_embed_dim=</span><span class="si">{default_embed_dim}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_attention</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_attention=</span><span class="si">{with_attention}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">with_cls_token</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;with_cls_token=</span><span class="si">{with_cls_token}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">shared_embed</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;shared_embed=</span><span class="si">{shared_embed}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;scale=</span><span class="si">{scale}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_standard</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;already_standard=</span><span class="si">{already_standard}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">quant_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
            <span class="p">)</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">([</span><span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">k</span><span class="si">}</span><span class="s2">&quot;</span> <span class="o">+</span> <span class="s2">&quot;=&quot;</span> <span class="o">+</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">v</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_args</span><span class="o">.</span><span class="n">items</span><span class="p">()])</span>
            <span class="p">)</span>
        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;ChunkTabPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">











  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">ChunkTextPreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor" href="#pytorch_widedeep.preprocessing.text_preprocessor.TextPreprocessor">TextPreprocessor</a></code></p>


        <p>Preprocessor to prepare the <code>deeptext</code> input dataset</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>text_col</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>column in the input dataframe containing either the texts or the
filenames where the text documents are stored</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>n_chunks</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Number of chunks that the text dataset is divided by.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>root_dir</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>If 'text_col' contains the filenames with the text documents, this is
the path to the directory where those documents are stored.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>max_vocab</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Maximum number of tokens in the vocabulary</p>
              </div>
            </td>
            <td>
                  <code>30000</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>min_freq</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Minimum frequency for a token to be part of the vocabulary</p>
              </div>
            </td>
            <td>
                  <code>5</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>maxlen</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Maximum length of the tokenized sequences</p>
              </div>
            </td>
            <td>
                  <code>80</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>pad_first</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Indicates whether the padding index will be added at the beginning or the
end of the sequences</p>
              </div>
            </td>
            <td>
                  <code>True</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>pad_idx</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>padding index. Fastai's Tokenizer leaves 0 for the 'unknown' token.</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>word_vectors_path</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Path to the pretrained word vectors</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>n_cpus</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>number of CPUs to used during the tokenization process</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>verbose</code>
            </td>
            <td>
                  <code>int</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Enable verbose output.</p>
              </div>
            </td>
            <td>
                  <code>1</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.vocab">vocab</span></code></td>
            <td>
                  <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.utils.fastai_transforms.Vocab" href="utils/fastai_transforms.html#pytorch_widedeep.utils.fastai_transforms.Vocab">Vocab</a></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>an instance of <code>pytorch_widedeep.utils.fastai_transforms.ChunkVocab</code></p>
              </div>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.text_preprocessor.ChunkTextPreprocessor.embedding_matrix">embedding_matrix</span></code></td>
            <td>
                  <code><span title="numpy.ndarray">ndarray</span></code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Array with the pretrained embeddings if <code>word_vectors_path</code> is not None</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Examples:</span></p>
    <div class="highlight"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">pandas</span> <span class="k">as</span> <span class="nn">pd</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">pytorch_widedeep.preprocessing</span> <span class="kn">import</span> <span class="n">ChunkTextPreprocessor</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">chunk_df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">({</span><span class="s1">&#39;text_column&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s2">&quot;life is like a box of chocolates&quot;</span><span class="p">,</span>
<span class="gp">... </span><span class="s2">&quot;You never know what you&#39;re gonna get&quot;</span><span class="p">]})</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">chunk_text_preprocessor</span> <span class="o">=</span> <span class="n">ChunkTextPreprocessor</span><span class="p">(</span><span class="n">text_col</span><span class="o">=</span><span class="s1">&#39;text_column&#39;</span><span class="p">,</span> <span class="n">n_chunks</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<span class="gp">... </span><span class="n">max_vocab</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span> <span class="n">min_freq</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">maxlen</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span> <span class="n">verbose</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">n_cpus</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">processed_chunk</span> <span class="o">=</span> <span class="n">chunk_text_preprocessor</span><span class="o">.</span><span class="n">fit_transform</span><span class="p">(</span><span class="n">chunk_df</span><span class="p">)</span>
</code></pre></div>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/text_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">264</span>
<span class="normal">265</span>
<span class="normal">266</span>
<span class="normal">267</span>
<span class="normal">268</span>
<span class="normal">269</span>
<span class="normal">270</span>
<span class="normal">271</span>
<span class="normal">272</span>
<span class="normal">273</span>
<span class="normal">274</span>
<span class="normal">275</span>
<span class="normal">276</span>
<span class="normal">277</span>
<span class="normal">278</span>
<span class="normal">279</span>
<span class="normal">280</span>
<span class="normal">281</span>
<span class="normal">282</span>
<span class="normal">283</span>
<span class="normal">284</span>
<span class="normal">285</span>
<span class="normal">286</span>
<span class="normal">287</span>
<span class="normal">288</span>
<span class="normal">289</span>
<span class="normal">290</span>
<span class="normal">291</span>
<span class="normal">292</span>
<span class="normal">293</span>
<span class="normal">294</span>
<span class="normal">295</span>
<span class="normal">296</span>
<span class="normal">297</span>
<span class="normal">298</span>
<span class="normal">299</span>
<span class="normal">300</span>
<span class="normal">301</span>
<span class="normal">302</span>
<span class="normal">303</span>
<span class="normal">304</span>
<span class="normal">305</span>
<span class="normal">306</span>
<span class="normal">307</span>
<span class="normal">308</span>
<span class="normal">309</span>
<span class="normal">310</span>
<span class="normal">311</span>
<span class="normal">312</span>
<span class="normal">313</span>
<span class="normal">314</span>
<span class="normal">315</span>
<span class="normal">316</span>
<span class="normal">317</span>
<span class="normal">318</span>
<span class="normal">319</span>
<span class="normal">320</span>
<span class="normal">321</span>
<span class="normal">322</span>
<span class="normal">323</span>
<span class="normal">324</span>
<span class="normal">325</span>
<span class="normal">326</span>
<span class="normal">327</span>
<span class="normal">328</span>
<span class="normal">329</span>
<span class="normal">330</span>
<span class="normal">331</span>
<span class="normal">332</span>
<span class="normal">333</span>
<span class="normal">334</span>
<span class="normal">335</span>
<span class="normal">336</span>
<span class="normal">337</span>
<span class="normal">338</span>
<span class="normal">339</span>
<span class="normal">340</span>
<span class="normal">341</span>
<span class="normal">342</span>
<span class="normal">343</span>
<span class="normal">344</span>
<span class="normal">345</span>
<span class="normal">346</span>
<span class="normal">347</span>
<span class="normal">348</span>
<span class="normal">349</span>
<span class="normal">350</span>
<span class="normal">351</span>
<span class="normal">352</span>
<span class="normal">353</span>
<span class="normal">354</span>
<span class="normal">355</span>
<span class="normal">356</span>
<span class="normal">357</span>
<span class="normal">358</span>
<span class="normal">359</span>
<span class="normal">360</span>
<span class="normal">361</span>
<span class="normal">362</span>
<span class="normal">363</span>
<span class="normal">364</span>
<span class="normal">365</span>
<span class="normal">366</span>
<span class="normal">367</span>
<span class="normal">368</span>
<span class="normal">369</span>
<span class="normal">370</span>
<span class="normal">371</span>
<span class="normal">372</span>
<span class="normal">373</span>
<span class="normal">374</span>
<span class="normal">375</span>
<span class="normal">376</span>
<span class="normal">377</span>
<span class="normal">378</span>
<span class="normal">379</span>
<span class="normal">380</span>
<span class="normal">381</span>
<span class="normal">382</span>
<span class="normal">383</span>
<span class="normal">384</span>
<span class="normal">385</span>
<span class="normal">386</span>
<span class="normal">387</span>
<span class="normal">388</span>
<span class="normal">389</span>
<span class="normal">390</span>
<span class="normal">391</span>
<span class="normal">392</span>
<span class="normal">393</span>
<span class="normal">394</span>
<span class="normal">395</span>
<span class="normal">396</span>
<span class="normal">397</span>
<span class="normal">398</span>
<span class="normal">399</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkTextPreprocessor</span><span class="p">(</span><span class="n">TextPreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Preprocessor to prepare the ``deeptext`` input dataset</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    text_col: str</span>
<span class="sd">        column in the input dataframe containing either the texts or the</span>
<span class="sd">        filenames where the text documents are stored</span>
<span class="sd">    n_chunks: int</span>
<span class="sd">        Number of chunks that the text dataset is divided by.</span>
<span class="sd">    root_dir: str, Optional, default = None</span>
<span class="sd">        If &#39;text_col&#39; contains the filenames with the text documents, this is</span>
<span class="sd">        the path to the directory where those documents are stored.</span>
<span class="sd">    max_vocab: int, default=30000</span>
<span class="sd">        Maximum number of tokens in the vocabulary</span>
<span class="sd">    min_freq: int, default=5</span>
<span class="sd">        Minimum frequency for a token to be part of the vocabulary</span>
<span class="sd">    maxlen: int, default=80</span>
<span class="sd">        Maximum length of the tokenized sequences</span>
<span class="sd">    pad_first: bool,  default = True</span>
<span class="sd">        Indicates whether the padding index will be added at the beginning or the</span>
<span class="sd">        end of the sequences</span>
<span class="sd">    pad_idx: int, default = 1</span>
<span class="sd">        padding index. Fastai&#39;s Tokenizer leaves 0 for the &#39;unknown&#39; token.</span>
<span class="sd">    word_vectors_path: str, Optional</span>
<span class="sd">        Path to the pretrained word vectors</span>
<span class="sd">    n_cpus: int, Optional, default = None</span>
<span class="sd">        number of CPUs to used during the tokenization process</span>
<span class="sd">    verbose: int, default 1</span>
<span class="sd">        Enable verbose output.</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    vocab: Vocab</span>
<span class="sd">        an instance of `pytorch_widedeep.utils.fastai_transforms.ChunkVocab`</span>
<span class="sd">    embedding_matrix: np.ndarray</span>
<span class="sd">        Array with the pretrained embeddings if `word_vectors_path` is not None</span>

<span class="sd">    Examples</span>
<span class="sd">    ---------</span>
<span class="sd">    &gt;&gt;&gt; import pandas as pd</span>
<span class="sd">    &gt;&gt;&gt; from pytorch_widedeep.preprocessing import ChunkTextPreprocessor</span>
<span class="sd">    &gt;&gt;&gt; chunk_df = pd.DataFrame({&#39;text_column&#39;: [&quot;life is like a box of chocolates&quot;,</span>
<span class="sd">    ... &quot;You never know what you&#39;re gonna get&quot;]})</span>
<span class="sd">    &gt;&gt;&gt; chunk_text_preprocessor = ChunkTextPreprocessor(text_col=&#39;text_column&#39;, n_chunks=1,</span>
<span class="sd">    ... max_vocab=25, min_freq=1, maxlen=10, verbose=0, n_cpus=1)</span>
<span class="sd">    &gt;&gt;&gt; processed_chunk = chunk_text_preprocessor.fit_transform(chunk_df)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">n_chunks</span><span class="p">:</span> <span class="nb">int</span><span class="p">,</span>
        <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">max_vocab</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">30000</span><span class="p">,</span>
        <span class="n">min_freq</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">5</span><span class="p">,</span>
        <span class="n">maxlen</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">80</span><span class="p">,</span>
        <span class="n">pad_first</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
        <span class="n">pad_idx</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
        <span class="n">already_processed</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
        <span class="n">word_vectors_path</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">n_cpus</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">verbose</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ChunkTextPreprocessor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
            <span class="n">text_col</span><span class="o">=</span><span class="n">text_col</span><span class="p">,</span>
            <span class="n">max_vocab</span><span class="o">=</span><span class="n">max_vocab</span><span class="p">,</span>
            <span class="n">min_freq</span><span class="o">=</span><span class="n">min_freq</span><span class="p">,</span>
            <span class="n">maxlen</span><span class="o">=</span><span class="n">maxlen</span><span class="p">,</span>
            <span class="n">pad_first</span><span class="o">=</span><span class="n">pad_first</span><span class="p">,</span>
            <span class="n">pad_idx</span><span class="o">=</span><span class="n">pad_idx</span><span class="p">,</span>
            <span class="n">already_processed</span><span class="o">=</span><span class="n">already_processed</span><span class="p">,</span>
            <span class="n">word_vectors_path</span><span class="o">=</span><span class="n">word_vectors_path</span><span class="p">,</span>
            <span class="n">n_cpus</span><span class="o">=</span><span class="n">n_cpus</span><span class="p">,</span>
            <span class="n">verbose</span><span class="o">=</span><span class="n">verbose</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="o">=</span> <span class="n">n_chunks</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">False</span>

    <span class="k">def</span> <span class="nf">partial_fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTextPreprocessor&quot;</span><span class="p">:</span>
        <span class="c1"># df is a chunk of the original dataframe</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">+=</span> <span class="mi">1</span>

        <span class="n">texts</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_read_texts</span><span class="p">(</span><span class="n">df</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="p">)</span>

        <span class="n">tokens</span> <span class="o">=</span> <span class="n">get_texts</span><span class="p">(</span><span class="n">texts</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">already_processed</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span><span class="p">)</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;vocab&quot;</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span> <span class="o">=</span> <span class="n">ChunkVocab</span><span class="p">(</span>
                <span class="n">max_vocab</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">max_vocab</span><span class="p">,</span>
                <span class="n">min_freq</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span><span class="p">,</span>
                <span class="n">pad_idx</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">pad_idx</span><span class="p">,</span>
                <span class="n">n_chunks</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">,</span>
            <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">tokens</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">chunk_counter</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
                <span class="nb">print</span><span class="p">(</span><span class="s2">&quot;The vocabulary contains </span><span class="si">{}</span><span class="s2"> tokens&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="o">.</span><span class="n">stoi</span><span class="p">)))</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">embedding_matrix</span> <span class="o">=</span> <span class="n">build_embeddings_matrix</span><span class="p">(</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">vocab</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_freq</span>
                <span class="p">)</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">is_fitted</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">return</span> <span class="bp">self</span>

    <span class="k">def</span> <span class="nf">fit</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">df</span><span class="p">:</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="s2">&quot;ChunkTextPreprocessor&quot;</span><span class="p">:</span>
        <span class="c1"># df is a chunk of the original dataframe</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">partial_fit</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="nb">str</span><span class="p">:</span>
        <span class="n">list_of_params</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;text_col=&#39;</span><span class="si">{text_col}</span><span class="s2">&#39;&quot;</span><span class="p">]</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_chunks</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_chunks=</span><span class="si">{n_chunks}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;root_dir=</span><span class="si">{root_dir}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;max_vocab=</span><span class="si">{max_vocab}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;min_freq=</span><span class="si">{min_freq}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;maxlen=</span><span class="si">{maxlen}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_first=</span><span class="si">{pad_first}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;pad_idx=</span><span class="si">{pad_idx}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">word_vectors_path</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;word_vectors_path=</span><span class="si">{word_vectors_path}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">n_cpus</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;n_cpus=</span><span class="si">{n_cpus}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">list_of_params</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="s2">&quot;verbose=</span><span class="si">{verbose}</span><span class="s2">&quot;</span><span class="p">)</span>
        <span class="n">all_params</span> <span class="o">=</span> <span class="s2">&quot;, &quot;</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">list_of_params</span><span class="p">)</span>
        <span class="k">return</span> <span class="sa">f</span><span class="s2">&quot;ChunkTextPreprocessor(</span><span class="si">{</span><span class="n">all_params</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="o">**</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span><span class="si">}</span><span class="s2">)&quot;</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">











  </div>

    </div>

</div>

<div class="doc doc-object doc-class">



<h2 id="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" class="doc doc-heading">
            <span class="doc doc-object-name doc-class-name">ChunkHFPreprocessor</span>


<a href="#pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor" class="headerlink" title="Permanent link">&para;</a></h2>


    <div class="doc doc-contents first">
            <p class="doc doc-class-bases">
              Bases: <code><a class="autorefs autorefs-internal" title="pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor" href="#pytorch_widedeep.preprocessing.hf_preprocessor.HFPreprocessor">HFPreprocessor</a></code></p>


        <p>Text processor to prepare the <code>deeptext</code> input dataset that is a
wrapper around HuggingFace's tokenizers.</p>
<p>Hugginface Tokenizer's are already 'trained'. Therefore, unlike the
<code>ChunkTextPreprocessor</code> this is mostly identical to the <code>HFPreprocessor</code>
with the only difference that the class needs a 'text_col' parameter to
be passed. Also the parameter <code>encode_params</code> is not really optional when
using this class. It must be passed containing at least the
'max_length' encoding parameter. This is because we need to ensure that
 all sequences have the same length when encoding in chunks.</p>


<p><span class="doc-section-title">Parameters:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
          <th>Default</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td>
                <code>model_name</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The model name from the transformers library e.g. <em>'bert-base-uncased'</em>.
Currently supported models are those from the families: BERT, RoBERTa,
DistilBERT, ALBERT and ELECTRA.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>text_col</code>
            </td>
            <td>
                  <code>str</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The column in the input dataframe containing the text data. When using
the <code>ChunkHFPreprocessor</code> the <code>text_col</code> parameter is mandatory.</p>
              </div>
            </td>
            <td>
                <em>required</em>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>root_dir</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[str]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>The root directory where the text files are located. This is only
needed if the text data is stored in text files. If the text data is
stored in a column in the input dataframe, this parameter is not
needed.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>use_fast_tokenizer</code>
            </td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Whether to use the fast tokenizer from HuggingFace or not</p>
              </div>
            </td>
            <td>
                  <code>True</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>num_workers</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[int]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Number of workers to use when preprocessing the text data. If not
None, and <code>use_fast_tokenizer</code> is False, the text data will be
preprocessed in parallel using the number of workers specified. If
<code>use_fast_tokenizer</code> is True, this argument is ignored.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>preprocessing_rules</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.List">List</span>[<span title="typing.Callable">Callable</span>[[str], str]]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>A list of functions to be applied to the text data before encoding.
This can be useful to clean the text data before encoding. For
example, removing html tags, special characters, etc.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>tokenizer_params</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Additional parameters to be passed to the HuggingFace's
<code>PreTrainedTokenizer</code>.</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
          <tr class="doc-section-item">
            <td>
                <code>encode_params</code>
            </td>
            <td>
                  <code><span title="typing.Optional">Optional</span>[<span title="typing.Dict">Dict</span>[str, <span title="typing.Any">Any</span>]]</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Additional parameters to be passed to the <code>batch_encode_plus</code> method
of the HuggingFace's <code>PreTrainedTokenizer</code>. In the case of the
<code>ChunkHFPreprocessor</code>, this parameter is not really <code>Optional</code>. It
must be passed containing at least the 'max_length' encoding
parameter</p>
              </div>
            </td>
            <td>
                  <code>None</code>
            </td>
          </tr>
      </tbody>
    </table>


<p><span class="doc-section-title">Attributes:</span></p>
    <table>
      <thead>
        <tr>
          <th>Name</th>
          <th>Type</th>
          <th>Description</th>
        </tr>
      </thead>
      <tbody>
          <tr class="doc-section-item">
            <td><code><span title="pytorch_widedeep.preprocessing.hf_preprocessor.ChunkHFPreprocessor.is_fitted">is_fitted</span></code></td>
            <td>
                  <code>bool</code>
            </td>
            <td>
              <div class="doc-md-description">
                <p>Boolean indicating if the preprocessor has been fitted. This is a
HuggingFacea tokenizer, so it is always considered fitted and this
attribute is manually set to True internally. This parameter exists
for consistency with the rest of the library and because is needed
for some functionality in the library.</p>
              </div>
            </td>
          </tr>
      </tbody>
    </table>






              <details class="quote">
                <summary>Source code in <code>pytorch_widedeep/preprocessing/hf_preprocessor.py</code></summary>
                <div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal">369</span>
<span class="normal">370</span>
<span class="normal">371</span>
<span class="normal">372</span>
<span class="normal">373</span>
<span class="normal">374</span>
<span class="normal">375</span>
<span class="normal">376</span>
<span class="normal">377</span>
<span class="normal">378</span>
<span class="normal">379</span>
<span class="normal">380</span>
<span class="normal">381</span>
<span class="normal">382</span>
<span class="normal">383</span>
<span class="normal">384</span>
<span class="normal">385</span>
<span class="normal">386</span>
<span class="normal">387</span>
<span class="normal">388</span>
<span class="normal">389</span>
<span class="normal">390</span>
<span class="normal">391</span>
<span class="normal">392</span>
<span class="normal">393</span>
<span class="normal">394</span>
<span class="normal">395</span>
<span class="normal">396</span>
<span class="normal">397</span>
<span class="normal">398</span>
<span class="normal">399</span>
<span class="normal">400</span>
<span class="normal">401</span>
<span class="normal">402</span>
<span class="normal">403</span>
<span class="normal">404</span>
<span class="normal">405</span>
<span class="normal">406</span>
<span class="normal">407</span>
<span class="normal">408</span>
<span class="normal">409</span>
<span class="normal">410</span>
<span class="normal">411</span>
<span class="normal">412</span>
<span class="normal">413</span>
<span class="normal">414</span>
<span class="normal">415</span>
<span class="normal">416</span>
<span class="normal">417</span>
<span class="normal">418</span>
<span class="normal">419</span>
<span class="normal">420</span>
<span class="normal">421</span>
<span class="normal">422</span>
<span class="normal">423</span>
<span class="normal">424</span>
<span class="normal">425</span>
<span class="normal">426</span>
<span class="normal">427</span>
<span class="normal">428</span>
<span class="normal">429</span>
<span class="normal">430</span>
<span class="normal">431</span>
<span class="normal">432</span>
<span class="normal">433</span>
<span class="normal">434</span>
<span class="normal">435</span>
<span class="normal">436</span>
<span class="normal">437</span>
<span class="normal">438</span>
<span class="normal">439</span>
<span class="normal">440</span>
<span class="normal">441</span>
<span class="normal">442</span>
<span class="normal">443</span>
<span class="normal">444</span>
<span class="normal">445</span>
<span class="normal">446</span>
<span class="normal">447</span>
<span class="normal">448</span>
<span class="normal">449</span>
<span class="normal">450</span>
<span class="normal">451</span>
<span class="normal">452</span>
<span class="normal">453</span>
<span class="normal">454</span>
<span class="normal">455</span>
<span class="normal">456</span>
<span class="normal">457</span>
<span class="normal">458</span>
<span class="normal">459</span>
<span class="normal">460</span>
<span class="normal">461</span>
<span class="normal">462</span>
<span class="normal">463</span>
<span class="normal">464</span>
<span class="normal">465</span>
<span class="normal">466</span>
<span class="normal">467</span>
<span class="normal">468</span>
<span class="normal">469</span>
<span class="normal">470</span>
<span class="normal">471</span>
<span class="normal">472</span>
<span class="normal">473</span>
<span class="normal">474</span></pre></div></td><td class="code"><div><pre><span></span><code><span class="k">class</span> <span class="nc">ChunkHFPreprocessor</span><span class="p">(</span><span class="n">HFPreprocessor</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;Text processor to prepare the ``deeptext`` input dataset that is a</span>
<span class="sd">    wrapper around HuggingFace&#39;s tokenizers.</span>

<span class="sd">    Hugginface Tokenizer&#39;s are already &#39;trained&#39;. Therefore, unlike the</span>
<span class="sd">    `ChunkTextPreprocessor` this is mostly identical to the `HFPreprocessor`</span>
<span class="sd">    with the only difference that the class needs a &#39;text_col&#39; parameter to</span>
<span class="sd">    be passed. Also the parameter `encode_params` is not really optional when</span>
<span class="sd">    using this class. It must be passed containing at least the</span>
<span class="sd">    &#39;max_length&#39; encoding parameter. This is because we need to ensure that</span>
<span class="sd">     all sequences have the same length when encoding in chunks.</span>

<span class="sd">    Parameters</span>
<span class="sd">    ----------</span>
<span class="sd">    model_name: str</span>
<span class="sd">        The model name from the transformers library e.g. _&#39;bert-base-uncased&#39;_.</span>
<span class="sd">        Currently supported models are those from the families: BERT, RoBERTa,</span>
<span class="sd">        DistilBERT, ALBERT and ELECTRA.</span>
<span class="sd">    text_col: str, default = None</span>
<span class="sd">        The column in the input dataframe containing the text data. When using</span>
<span class="sd">        the `ChunkHFPreprocessor` the `text_col` parameter is mandatory.</span>
<span class="sd">    root_dir: Optional[str], default = None</span>
<span class="sd">        The root directory where the text files are located. This is only</span>
<span class="sd">        needed if the text data is stored in text files. If the text data is</span>
<span class="sd">        stored in a column in the input dataframe, this parameter is not</span>
<span class="sd">        needed.</span>
<span class="sd">    use_fast_tokenizer: bool, default = False</span>
<span class="sd">        Whether to use the fast tokenizer from HuggingFace or not</span>
<span class="sd">    num_workers: Optional[int], default = None</span>
<span class="sd">        Number of workers to use when preprocessing the text data. If not</span>
<span class="sd">        None, and `use_fast_tokenizer` is False, the text data will be</span>
<span class="sd">        preprocessed in parallel using the number of workers specified. If</span>
<span class="sd">        `use_fast_tokenizer` is True, this argument is ignored.</span>
<span class="sd">    preprocessing_rules: Optional[List[Callable[[str], str]]], default = None</span>
<span class="sd">        A list of functions to be applied to the text data before encoding.</span>
<span class="sd">        This can be useful to clean the text data before encoding. For</span>
<span class="sd">        example, removing html tags, special characters, etc.</span>
<span class="sd">    tokenizer_params: Optional[Dict[str, Any]], default = None</span>
<span class="sd">        Additional parameters to be passed to the HuggingFace&#39;s</span>
<span class="sd">        `PreTrainedTokenizer`.</span>
<span class="sd">    encode_params: Optional[Dict[str, Any]], default = None</span>
<span class="sd">        Additional parameters to be passed to the `batch_encode_plus` method</span>
<span class="sd">        of the HuggingFace&#39;s `PreTrainedTokenizer`. In the case of the</span>
<span class="sd">        `ChunkHFPreprocessor`, this parameter is not really `Optional`. It</span>
<span class="sd">        must be passed containing at least the &#39;max_length&#39; encoding</span>
<span class="sd">        parameter</span>

<span class="sd">    Attributes</span>
<span class="sd">    ----------</span>
<span class="sd">    is_fitted: bool</span>
<span class="sd">        Boolean indicating if the preprocessor has been fitted. This is a</span>
<span class="sd">        HuggingFacea tokenizer, so it is always considered fitted and this</span>
<span class="sd">        attribute is manually set to True internally. This parameter exists</span>
<span class="sd">        for consistency with the rest of the library and because is needed</span>
<span class="sd">        for some functionality in the library.</span>

<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
        <span class="bp">self</span><span class="p">,</span>
        <span class="n">model_name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="o">*</span><span class="p">,</span>
        <span class="n">text_col</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">root_dir</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">use_fast_tokenizer</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
        <span class="n">num_workers</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">preprocessing_rules</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="n">Callable</span><span class="p">[[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">tokenizer_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
        <span class="n">encode_params</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="p">):</span>
        <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span>
            <span class="n">model_name</span><span class="o">=</span><span class="n">model_name</span><span class="p">,</span>
            <span class="n">use_fast_tokenizer</span><span class="o">=</span><span class="n">use_fast_tokenizer</span><span class="p">,</span>
            <span class="n">text_col</span><span class="o">=</span><span class="n">text_col</span><span class="p">,</span>
            <span class="n">num_workers</span><span class="o">=</span><span class="n">num_workers</span><span class="p">,</span>
            <span class="n">preprocessing_rules</span><span class="o">=</span><span class="n">preprocessing_rules</span><span class="p">,</span>
            <span class="n">tokenizer_params</span><span class="o">=</span><span class="n">tokenizer_params</span><span class="p">,</span>
            <span class="n">encode_params</span><span class="o">=</span><span class="n">encode_params</span><span class="p">,</span>
        <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span> <span class="o">=</span> <span class="n">root_dir</span>

        <span class="c1"># when using in chunks encode_params is not really optional. I will</span>
        <span class="c1"># review types in due time</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s2">&quot;The &#39;encode_params&#39; dict must be passed to the ChunkHFTokenizer &quot;</span>
                <span class="s2">&quot;containing at least the &#39;max_length&#39; encoding parameter&quot;</span>
            <span class="p">)</span>

        <span class="k">if</span> <span class="s2">&quot;padding&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span> <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;padding&quot;</span><span class="p">]:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;padding&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">if</span> <span class="p">(</span>
            <span class="s2">&quot;truncation&quot;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span>
            <span class="ow">or</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;truncation&quot;</span><span class="p">]</span>
        <span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="p">[</span><span class="s2">&quot;truncation&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="kc">True</span>

    <span class="k">def</span> <span class="fm">__repr__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">(</span>
            <span class="sa">f</span><span class="s2">&quot;ChunkHFPreprocessor(text_col=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">text_col</span><span class="si">}</span><span class="s2">, model_name=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">model_name</span><span class="si">}</span><span class="s2">, &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;use_fast_tokenizer=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">use_fast_tokenizer</span><span class="si">}</span><span class="s2">, num_workers=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">num_workers</span><span class="si">}</span><span class="s2">, &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;preprocessing_rules=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">preprocessing_rules</span><span class="si">}</span><span class="s2">, tokenizer_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">tokenizer_params</span><span class="si">}</span><span class="s2">, &quot;</span>
            <span class="sa">f</span><span class="s2">&quot;encode_params=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">encode_params</span><span class="si">}</span><span class="s2">, root_dir=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">root_dir</span><span class="si">}</span><span class="s2">)&quot;</span>
        <span class="p">)</span>
</code></pre></div></td></tr></table></div>
              </details>



  <div class="doc doc-children">











  </div>

    </div>

</div>








  <aside class="md-source-file">
    
    
    
      
  
  <span class="md-source-file__fact">
    <span class="md-icon" title="Contributors">
      
        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 5.5A3.5 3.5 0 0 1 15.5 9a3.5 3.5 0 0 1-3.5 3.5A3.5 3.5 0 0 1 8.5 9 3.5 3.5 0 0 1 12 5.5M5 8c.56 0 1.08.15 1.53.42-.15 1.43.27 2.85 1.13 3.96C7.16 13.34 6.16 14 5 14a3 3 0 0 1-3-3 3 3 0 0 1 3-3m14 0a3 3 0 0 1 3 3 3 3 0 0 1-3 3c-1.16 0-2.16-.66-2.66-1.62a5.54 5.54 0 0 0 1.13-3.96c.45-.27.97-.42 1.53-.42M5.5 18.25c0-2.07 2.91-3.75 6.5-3.75s6.5 1.68 6.5 3.75V20h-13zM0 20v-1.5c0-1.39 1.89-2.56 4.45-2.9-.59.68-.95 1.62-.95 2.65V20zm24 0h-3.5v-1.75c0-1.03-.36-1.97-.95-2.65 2.56.34 4.45 1.51 4.45 2.9z"/></svg>
      
    </span>
    <nav>
      
        <a href="mailto:jrzaurin@gmail.com">Javier</a>, 
        <a href="mailto:mulinka.pavol@gmail.com">Pavol Mulinka</a>
    </nav>
  </span>

    
    
  </aside>





                
              </article>
            </div>
          
          
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>
        
      </main>
      
        <footer class="md-footer">
  
  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">
  
    <div class="md-copyright__highlight">
      Javier Zaurin and Pavol Mulinka
    </div>
  
  
    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>
  
</div>
      
        <div class="md-social">
  
    
    
    
    
      
      
    
    <a href="https://jrzaurin.medium.com/" target="_blank" rel="noopener" title="jrzaurin.medium.com" class="md-social__link">
      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.6.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2024 Fonticons, Inc.--><path d="M180.5 74.262C80.813 74.262 0 155.633 0 256s80.819 181.738 180.5 181.738S361 356.373 361 256 280.191 74.262 180.5 74.262m288.25 10.646c-49.845 0-90.245 76.619-90.245 171.095s40.406 171.1 90.251 171.1 90.251-76.619 90.251-171.1H559c0-94.503-40.4-171.095-90.248-171.095Zm139.506 17.821c-17.526 0-31.735 68.628-31.735 153.274s14.2 153.274 31.735 153.274S640 340.631 640 256c0-84.649-14.215-153.271-31.742-153.271Z"/></svg>
    </a>
  
</div>
      
    </div>
  </div>
</footer>
      
    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>
    
    
    <script id="__config" type="application/json">{"base": "..", "features": ["navigation.tabs", "navigation.tabs.sticky", "navigation.indexes", "navigation.expand", "toc.integrate"], "search": "../assets/javascripts/workers/search.6ce7567c.min.js", "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}}</script>
    
    
      <script src="../assets/javascripts/bundle.83f73b43.min.js"></script>
      
        <script src="../stylesheets/extra.js"></script>
      
        <script src="../javascripts/mathjax.js"></script>
      
        <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
      
        <script src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
      
    
  </body>
</html>